Kouhei Sutou
null+****@clear*****
Fri Mar 29 16:33:56 JST 2013
Kouhei Sutou 2013-03-29 16:33:56 +0900 (Fri, 29 Mar 2013) New Revision: b6223da03af987135d09cfe6152645f0c61cf7c7 https://github.com/groonga/fluent-plugin-droonga/commit/b6223da03af987135d09cfe6152645f0c61cf7c7 Message: Implement handler plugin feature #1 Added files: lib/droonga/handler.rb lib/droonga/handler_plugin.rb lib/droonga/plugin.rb Copied files: lib/droonga/plugin/handler_search.rb (from lib/droonga/worker.rb) Modified files: lib/droonga/worker.rb lib/fluent/plugin/out_droonga.rb sample/fluentd.conf test/test_worker.rb Added: lib/droonga/handler.rb (+53 -0) 100644 =================================================================== --- /dev/null +++ lib/droonga/handler.rb 2013-03-29 16:33:56 +0900 (bc12182) @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2013 droonga project +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +require "droonga/handler_plugin" + +module Droonga + class Handler + @@commands = {} + class << self + def command(name_or_map) + if name_or_map.is_a?(Hash) + command_map = name_or_map + command_map.each do |command_name, method_name| + @@commands[command_name.to_s] = method_name + end + else + name = name_or_map + method_name = name + @@commands[name.to_s] = method_name + end + end + end + + def initialize(context) + @context = context + end + + def shutdown + end + + def handlable?(command) + not @@commands[command.to_s].nil? + end + + def handle(command, request) + __send__(@@commands[command], request) + end + end +end Added: lib/droonga/handler_plugin.rb (+35 -0) 100644 =================================================================== --- /dev/null +++ lib/droonga/handler_plugin.rb 2013-03-29 16:33:56 +0900 (5aaa1cb) @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2013 droonga project +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +module Droonga + class HandlerPlugin + @@plugins = {} + class << self + def register(name, handler_class) + @@plugins[name] = handler_class + end + end + + def initialize(name) + @name = name + end + + def instantiate(*args) + @@plugins[@name].new(*args) + end + end +end Added: lib/droonga/plugin.rb (+41 -0) 100644 =================================================================== --- /dev/null +++ lib/droonga/plugin.rb 2013-03-29 16:33:56 +0900 (6d58b26) @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2013 droonga project +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +module Droonga + class Plugin + class << self + def load_all + $LOAD_PATH.each do |load_path| + Dir.glob("#{load_path}/droonga/plugin/*_*.rb") do |path| + type, name = File.basename(path, ".rb").split(/_/, 2) + plugin = new(type, name) + plugin.load + end + end + end + end + + def initialize(type, name) + @type = type + @name = name + end + + def load + require "droonga/plugin/#{@type}_#{@name}" + end + end +end Copied: lib/droonga/plugin/handler_search.rb (+28 -44) 75% =================================================================== --- lib/droonga/worker.rb 2013-03-28 23:13:22 +0900 (f165c0d) +++ lib/droonga/plugin/handler_search.rb 2013-03-29 16:33:56 +0900 (8e34e7a) @@ -15,53 +15,16 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "time" -require 'tsort' -require 'groonga' +require "tsort" +require "groonga" -module Droonga - class QuerySorter - include TSort - def initialize() - @queries = {} - end - - def add(name, sources=[]) - @queries[name] = sources - end - - def tsort_each_node(&block) - @queries.each_key(&block) - end - - def tsort_each_child(node, &block) - if @queries[node] - @queries[node].each(&block) - end - end - end - - class Worker - def initialize(database, queue_name) - @context = Groonga::Context.new - @database =****@conte*****_database(database) - @queue_name = queue_name - end - - def shutdown - @database.close - @context.close - @database = @context = nil - end +require "droonga/handler" - def process_message(envelope) - case envelope["type"] - when "search" - search(envelope["body"]) - end - end +module Droonga + class SearchHandler < Droonga::Handler + Droonga::HandlerPlugin.register("search", self) - private + command :search def search(request) queries = request["queries"] results = {} @@ -116,5 +79,26 @@ module Droonga } end end + + class QuerySorter + include TSort + def initialize() + @queries = {} + end + + def add(name, sources=[]) + @queries[name] = sources + end + + def tsort_each_node(&block) + @queries.each_key(&block) + end + + def tsort_each_child(node, &block) + if @queries[node] + @queries[node].each(&block) + end + end + end end end Modified: lib/droonga/worker.rb (+16 -79) =================================================================== --- lib/droonga/worker.rb 2013-03-28 23:13:22 +0900 (f165c0d) +++ lib/droonga/worker.rb 2013-03-29 16:33:56 +0900 (fd8561a) @@ -15,105 +15,42 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "time" -require 'tsort' require 'groonga' +require "droonga/handler_plugin" module Droonga - class QuerySorter - include TSort - def initialize() - @queries = {} - end - - def add(name, sources=[]) - @queries[name] = sources - end - - def tsort_each_node(&block) - @queries.each_key(&block) - end - - def tsort_each_child(node, &block) - if @queries[node] - @queries[node].each(&block) - end - end - end - class Worker def initialize(database, queue_name) @context = Groonga::Context.new @database =****@conte*****_database(database) @queue_name = queue_name + @handlers = [] + end + + def add_handler(name) + plugin = HandlerPlugin.new(name) + @handlers << plugin.instantiate(@context) end def shutdown + @handlers.each do |handler| + handler.shutdown + end @database.close @context.close @database = @context = nil end def process_message(envelope) - case envelope["type"] - when "search" - search(envelope["body"]) - end + command = envelope["type"] + handler = find_handler(command) + handler.handle(command, envelope["body"]) end private - def search(request) - queries = request["queries"] - results = {} - outputs = {} - query_sorter = QuerySorter.new - queries.each do |name, query| - query_sorter.add(name, [query["source"]]) - end - query_sorter.tsort.each do |name| - if queries[name] - search_query(name, queries, results, outputs) - elsif @context[name] - results[name] = @context[name] - else - raise "undefined source(#{name}) was assigned" - end - end - outputs - end - - def search_query(name, queries, results, outputs) - start_time = Time.now - query = queries[name] - source = results[query["source"]] - if query["output"] - offset = query["offset"] || 0 - limit = query["limit"] || 10 - columns = source.columns - attributes = columns.collect do |column| - { - "name" => column.local_name, - "type" => column.range.name, - "vector" => column.vector?, - } - end - column_names = columns.collect(&:local_name) - records = source.open_cursor(:offset => offset, - :limit => limit) do |cursor| - cursor.collect do |record| - column_names.collect do |name| - record[name] - end - end - end - elapsed_time = Time.now.to_f - start_time.to_f - outputs[name] = { - "count" => source.size, - "startTime" => start_time.iso8601, - "elapsedTime" => elapsed_time, - "attributes" => attributes, - "records" => records, - } + def find_handler(command) + @handlers.find do |handler| + handler.handlable?(command) end end end Modified: lib/fluent/plugin/out_droonga.rb (+21 -1) =================================================================== --- lib/fluent/plugin/out_droonga.rb 2013-03-28 23:13:22 +0900 (bf0d4fa) +++ lib/fluent/plugin/out_droonga.rb 2013-03-29 16:33:56 +0900 (f5ac822) @@ -17,6 +17,7 @@ require "fluent-logger" require "droonga/worker" +require "droonga/plugin" module Fluent class DroongaOutput < Output @@ -24,6 +25,14 @@ module Fluent config_param :database, :string, :default => "droonga.db" config_param :queue_name, :string, :default => "DroongaQueue" + config_param :handlers, :default => [] do |value| + value.split(/\s*,\s*/) + end + + def configure(conf) + super + load_handlers + end def start super @@ -107,8 +116,19 @@ module Fluent end private + def load_handlers + @handlers.each do |handler_name| + plugin = Droonga::Plugin.new("handler", handler_name) + plugin.load + end + end + def create_worker - Droonga::Worker.new(@database, @queue_name) + worker = Droonga::Worker.new(@database, @queue_name) + @handlers.each do |handler_name| + worker.add_handler(handler_name) + end + worker end def create_logger(tag, options) Modified: sample/fluentd.conf (+1 -0) =================================================================== --- sample/fluentd.conf 2013-03-28 23:13:22 +0900 (3665d5b) +++ sample/fluentd.conf 2013-03-29 16:33:56 +0900 (21b8fbb) @@ -4,4 +4,5 @@ <match droonga.message> type droonga + handlers search </match> Modified: test/test_worker.rb (+9 -0) =================================================================== --- test/test_worker.rb 2013-03-28 23:13:22 +0900 (dffaa1e) +++ test/test_worker.rb 2013-03-29 16:33:56 +0900 (6246336) @@ -20,6 +20,7 @@ require "droonga/worker" class WorkerTest < Test::Unit::TestCase def setup setup_database + setup_handlers setup_worker end @@ -32,8 +33,16 @@ class WorkerTest < Test::Unit::TestCase restore(fixture_data("document.grn")) end + def setup_handlers + ["search"].each do |handler_name| + plugin = Droonga::Plugin.new("handler", handler_name) + plugin.load + end + end + def setup_worker @worker = Droonga::Worker.new(@database_path.to_s, "DroongaQueue") + @worker.add_handler("search") end def teardown_worker -------------- next part -------------- HTML����������������������������...Download