Kouhei Sutou
null+****@clear*****
Mon Jan 15 15:09:33 JST 2018
Kouhei Sutou 2018-01-15 15:09:33 +0900 (Mon, 15 Jan 2018) New Revision: 13e60b6e12a998b109aeeec08970087b03a7916a https://github.com/groonga/groonga-log/commit/13e60b6e12a998b109aeeec08970087b03a7916a Message: Support parsing compressed logs Added files: lib/groonga-log/input.rb test/fixture/groonga.log test/fixture/groonga.log.gz test/fixture/groonga.log.zip test/test-input.rb Modified files: groonga-log.gemspec lib/groonga-log/parser.rb test/helper.rb test/run-test.rb test/test-parser.rb Modified: groonga-log.gemspec (+7 -6) =================================================================== --- groonga-log.gemspec 2018-01-15 14:35:28 +0900 (8f2efa3) +++ groonga-log.gemspec 2018-01-15 15:09:33 +0900 (f1d0faf) @@ -1,6 +1,7 @@ -# -*- mode: ruby; coding: utf-8 -*- +# -*- ruby -*- # # Copyright (C) 2017 Yasuhiro Horimoto <horimoto �� clear-code.com> +# Copyright (C) 2018 Kouhei Sutou <kou �� clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -28,8 +29,8 @@ Gem::Specification.new do |spec| spec.name = "groonga-log" spec.version = GroongaLog::VERSION - spec.authors = ["Horimoto Yasuhiro"] - spec.email = ["horimoto �� clear-code.com"] + spec.authors = ["Horimoto Yasuhiro", "Kouhei Sutou"] + spec.email = ["horimoto �� clear-code.com", "kou �� clear-code.com"] readme = File.read("README.md", :encoding => "UTF-8") entries = readme.split(/^\#\#\s(.*)$/) @@ -44,9 +45,9 @@ Gem::Specification.new do |spec| spec.licenses = ["LGPLv2.1+"] spec.require_paths = ["lib"] - spec.add_development_dependency "bundler", "~> 1.13" - spec.add_development_dependency "rake", "~> 10.0" + spec.add_runtime_dependency "archive-zip" + spec.add_development_dependency("bundler") + spec.add_development_dependency("rake") spec.add_development_dependency("test-unit") - spec.add_development_dependency("test-unit-notify") end Added: lib/groonga-log/input.rb (+74 -0) 100644 =================================================================== --- /dev/null +++ lib/groonga-log/input.rb 2018-01-15 15:09:33 +0900 (f73f688) @@ -0,0 +1,74 @@ +# Copyright (C) 2018 Kouhei Sutou <kou �� clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +require "zlib" + +require "archive/zip" + +module GroongaLog + class Input + class << self + def open(path) + input = new(path) + if block_given? + begin + yield(input) + ensure + input.close unless input.closed? + end + else + input + end + end + end + + def initialize(path) + @path = path + @path =****@path*****_path if****@path*****_to?(:to_path) + + case File.extname(@path).downcase + when ".gz" + @raw = Zlib::GzipReader.open(@path) + when ".zip" + @raw = Archive::Zip.new(@path, :r) + else + @raw = File.new(@path) + end + end + + def each_line(&block) + return to_enum(__method__) unless block_given? + + case @raw + when Archive::Zip + @raw.each do |entry| + next unless entry.file? + entry.file_data.each_line(&block) + end + else + @raw.each_line(&block) + end + end + + def close + @raw.close + end + + def closed? + @raw.closed? + end + end +end Modified: lib/groonga-log/parser.rb (+18 -5) =================================================================== --- lib/groonga-log/parser.rb 2018-01-15 14:35:28 +0900 (902e525) +++ lib/groonga-log/parser.rb 2018-01-15 15:09:33 +0900 (3a5ed91) @@ -1,5 +1,6 @@ -# Copyright (C) 2017 Yasuhiro Horimoto <horimoto �� clear-code.com> -# Copyright (C) 2017 Kentaro Hayashi <hayashi �� clear-code.com> +# Copyright (C) 2017 Yasuhiro Horimoto <horimoto �� clear-code.com> +# Copyright (C) 2017 Kentaro Hayashi <hayashi �� clear-code.com> +# Copyright (C) 2018 Kouhei Sutou <kou �� clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,6 +17,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA require "groonga-log/entry" +require "groonga-log/input" module GroongaLog class Parser @@ -31,7 +33,7 @@ module GroongaLog class << self def target_line?(line) if line.respond_to?(:valid_encoding?) - return false unless line.valid_eocoding? + return false unless line.valid_encoding? end return false unless PATTERN.match(line) @@ -50,6 +52,17 @@ module GroongaLog end end end + + def filter_paths(paths) + paths.reject do |path| + case File.extname(path).downcase + when ".zip", ".gz" # TODO: support decompress + true + else + false + end + end + end end def parse(input) @@ -83,9 +96,9 @@ module GroongaLog def parse_paths(paths, &block) return to_enum(__method__, paths) unless block_given? - target_paths = self.class.sort_paths(filter_paths(paths)) + target_paths = self.class.sort_paths(paths) target_paths.each do |path| - File.open(path) do |log| + Input.open(path) do |log| parse(log, &block) end end Added: test/fixture/groonga.log (+2 -0) 100644 =================================================================== --- /dev/null +++ test/fixture/groonga.log 2018-01-15 15:09:33 +0900 (e5125d9) @@ -0,0 +1,2 @@ +2018-01-15 15:01:23.305389|n| grn_init: <7.1.0-1-gef1bd38> +2018-01-15 15:01:23.712541|n| grn_fin (0) Added: test/fixture/groonga.log.gz (+0 -0) 100644 =================================================================== (Binary files differ) Added: test/fixture/groonga.log.zip (+0 -0) 100644 =================================================================== (Binary files differ) Modified: test/helper.rb (+12 -1) =================================================================== --- test/helper.rb 2018-01-15 14:35:28 +0900 (af4cfd7) +++ test/helper.rb 2018-01-15 15:09:33 +0900 (3318d75) @@ -1,4 +1,5 @@ -# Copyright (C) 2017 Kentaro Hayashi <hayashi �� clear-code.com> +# Copyright (C) 2017 Kentaro Hayashi <hayashi �� clear-code.com> +# Copyright (C) 2018 Kouhei Sutou <kou �� clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -14,4 +15,14 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +require "pathname" + require "groonga-log" + +module Helper + module Fixture + def fixture_path(*components) + Pathname.new(__dir__).join("fixture").join(*components) + end + end +end Modified: test/run-test.rb (+4 -11) =================================================================== --- test/run-test.rb 2018-01-15 14:35:28 +0900 (bb088b9) +++ test/run-test.rb 2018-01-15 15:09:33 +0900 (55d2256) @@ -1,6 +1,6 @@ #!/usr/bin/env ruby # -# Copyright (C) 2012-2013 Kouhei Sutou <kou �� clear-code.com> +# Copyright (C) 2012-2018 Kouhei Sutou <kou �� clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -26,20 +26,13 @@ top_dir = base_dir.parent lib_dir = base_dir + "lib" test_dir = base_dir + "test" -require "test-unit" -require "test/unit/notify" +require "test/unit" Test::Unit::Priority.enable $LOAD_PATH.unshift(lib_dir.to_s) -$LOAD_PATH.unshift(test_dir.to_s) -require "helper" +require_relative "helper" -Dir.glob("#{base_dir}/test/**/test{_,-}*.rb") do |file| - require file.sub(/\.rb\z/, '') -end -ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000" - -exit Test::Unit::AutoRunner.run +exit(Test::Unit::AutoRunner.run(true, test_dir)) Added: test/test-input.rb (+81 -0) 100644 =================================================================== --- /dev/null +++ test/test-input.rb 2018-01-15 15:09:33 +0900 (6af2bd0) @@ -0,0 +1,81 @@ +# Copyright (C) 2018 Kouhei Sutou <kou �� clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +class ParserTest < Test::Unit::TestCase + include Helper::Fixture + + def parse(input) + parser = GroongaLog::Parser.new + parser.parse(input).collect(&:to_h) + end + + def test_text + input = GroongaLog::Input.new(fixture_path("groonga.log")) + raw_entries = [ + { + :timestamp => Time.local(2018, 1, 15, 15, 1, 23, 305389), + :log_level => :notice, + :pid => nil, + :message => "grn_init: <7.1.0-1-gef1bd38>", + }, + { + :timestamp => Time.local(2018, 1, 15, 15, 1, 23, 712541), + :log_level => :notice, + :pid => nil, + :message => "grn_fin (0)", + }, + ] + assert_equal(raw_entries, parse(input)) + end + + def test_gzip + input = GroongaLog::Input.new(fixture_path("groonga.log.gz")) + raw_entries = [ + { + :timestamp => Time.local(2018, 1, 15, 15, 1, 23, 305389), + :log_level => :notice, + :pid => nil, + :message => "grn_init: <7.1.0-1-gef1bd38>", + }, + { + :timestamp => Time.local(2018, 1, 15, 15, 1, 23, 712541), + :log_level => :notice, + :pid => nil, + :message => "grn_fin (0)", + }, + ] + assert_equal(raw_entries, parse(input)) + end + + def test_zip + input = GroongaLog::Input.new(fixture_path("groonga.log.zip")) + raw_entries = [ + { + :timestamp => Time.local(2018, 1, 15, 15, 1, 23, 305389), + :log_level => :notice, + :pid => nil, + :message => "grn_init: <7.1.0-1-gef1bd38>", + }, + { + :timestamp => Time.local(2018, 1, 15, 15, 1, 23, 712541), + :log_level => :notice, + :pid => nil, + :message => "grn_fin (0)", + }, + ] + assert_equal(raw_entries, parse(input)) + end +end Modified: test/test-parser.rb (+1 -3) =================================================================== --- test/test-parser.rb 2018-01-15 14:35:28 +0900 (32e15be) +++ test/test-parser.rb 2018-01-15 15:09:33 +0900 (1a94862) @@ -16,9 +16,7 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -require "helper" - -class ParserTest < Test::Unit::TestCase +class InputTest < Test::Unit::TestCase sub_test_case("extract fields") do def test_with_pid raw_entry = { -------------- next part -------------- HTML����������������������������... URL: https://lists.osdn.me/mailman/archives/groonga-commit/attachments/20180115/4dfb6732/attachment-0001.htm
