Kouhei Sutou
null+****@clear*****
Fri Feb 7 14:38:21 JST 2014
Kouhei Sutou 2014-02-07 14:38:21 +0900 (Fri, 07 Feb 2014) New Revision: 06e2b102ab48d0bae4d05e7b61f033684ac55aac https://github.com/droonga/fluent-plugin-droonga/commit/06e2b102ab48d0bae4d05e7b61f033684ac55aac Message: Unify InputAdapter and OutputAdapter to Adapter Location is changed. Before: lib/plugin/input_adapter/NAME.rb lib/plugin/output_adapter/NAME.rb After: lib/plugin/NAME.rb # <- This file has adapter Registration API is changed. Before: class MyInputAdapter < Droonga::InputAdapter repository.register("my-plugin", self) command :my_command def my_command(input_message) end end class MyOutputAdapter < Droonga::OutputAdapter repository.register("my-plugin", self) command :my_command, :pattern => ["replyTo.type", :equal, "my_command.result"] def my_command(output_message) end end After: module MyPlugin class Adapter < Droonga::Adapter plugin.name = "my-plugin" message.input_pattern = ["type", :equal, "select"] def adapt_input(input_message) end def adapt_output(output_message) end end end catalog.json is changed. Before: { ..., "input_adapter": { "plugins": ["my-plugin"] }, "output_adapter": { "plugins": ["my-plugin"] } } After: { ... "datasets": { "DATASET": { ..., "plugins": ["my-plugin"] } } } Added files: lib/droonga/plugins/error.rb lib/droonga/plugins/groonga/select.rb test/unit/plugins/groonga/select/test_adapter_input.rb Copied files: lib/droonga/adapter.rb (from lib/droonga/catalog/version1.rb) lib/droonga/plugins/crud.rb (from lib/droonga/plugin/output_adapter/crud.rb) Removed files: lib/droonga/plugin/input_adapter/groonga/select.rb lib/droonga/plugin/output_adapter/groonga.rb lib/droonga/plugin/output_adapter/groonga/select.rb test/unit/plugin/input_adapter/groonga/test_select.rb Modified files: lib/droonga/adapter_runner.rb lib/droonga/catalog/base.rb lib/droonga/catalog/version1.rb lib/droonga/dispatcher.rb lib/droonga/plugin_loader.rb sample/cluster/catalog.json test/command/config/default/catalog.json test/unit/catalog/test_version1.rb Renamed files: lib/droonga/adapter/message_configuration.rb (from test/unit/output_adapter/test_options.rb) lib/droonga/adapter/plugin_configuration.rb (from test/unit/input_adapter/test_options.rb) lib/droonga/plugins/groonga.rb (from lib/droonga/plugin/input_adapter/groonga.rb) lib/droonga/plugins/groonga/generic.rb (from lib/droonga/plugin/output_adapter/crud.rb) test/unit/plugins/groonga/select/test_adapter_output.rb (from test/unit/plugin/output_adapter/groonga/test_select.rb) Copied: lib/droonga/adapter.rb (+30 -15) 54% =================================================================== --- lib/droonga/catalog/version1.rb 2014-02-07 13:59:19 +0900 (53df3e8) +++ lib/droonga/adapter.rb 2014-02-07 14:38:21 +0900 (02352f5) @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Droonga Project +# Copyright (C) 2014 Droonga Project # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,27 +13,42 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/catalog/base" - module Droonga - module Catalog - class Version1 < Base - def initialize(*args) + class Adapter + class << self + def adapter_classes + @@adapter_classes ||= [] + end + + def inherited(sub_class) super - normalize_input_adapter - normalize_output_adapter + adapter_classes << sub_class + end + + def plugin + PluginConfiguration.new(self) + end + + def message + MessageConfiguration.new(self) end - private - def normalize_input_adapter - @data["input_adapter"] ||= {} - @data["input_adapter"]["plugins"] ||= @options["plugins"] + def id + options[:id] || name || object_id.to_s end - def normalize_output_adapter - @data["output_adapter"] ||= {} - @data["output_adapter"]["plugins"] ||= @options["plugins"] + def options + @options ||= {} end end + + def adapt_input(input_message) + end + + def adapt_output(output_message) + end end end + +require "droonga/adapter/plugin_configuration" +require "droonga/adapter/message_configuration" Renamed: lib/droonga/adapter/message_configuration.rb (+23 -14) 55% =================================================================== --- test/unit/output_adapter/test_options.rb 2014-02-07 13:59:19 +0900 (a0816e5) +++ lib/droonga/adapter/message_configuration.rb 2014-02-07 14:38:21 +0900 (bc005b6) @@ -13,24 +13,33 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/output_adapter_options" +module Droonga + class Adapter + class MessageConfiguration + def initialize(adapter_class) + @adapter_class = adapter_class + end -class OutputAdapterOptionsTest < Test::Unit::TestCase - def options(data) - Droonga::OutputAdapterOptions.new(data) - end + def input_pattern + configuration[:input_pattern] + end - class PluginsTest < self - def plugins(data) - options(data).plugins - end + def input_pattern=(pattern) + configuration[:input_pattern] = pattern + end - def test_nothing - assert_equal([], plugins({})) - end + def output_pattern + configuration[:output_pattern] + end + + def output_pattern=(pattern) + configuration[:output_pattern] = pattern + end - def test_have_values - assert_equal(["groonga"], plugins("plugins" => ["groonga"])) + private + def configuration + @adapter_class.options[:message] ||= {} + end end end end Renamed: lib/droonga/adapter/plugin_configuration.rb (+16 -15) 64% =================================================================== --- test/unit/input_adapter/test_options.rb 2014-02-07 13:59:19 +0900 (0fa0342) +++ lib/droonga/adapter/plugin_configuration.rb 2014-02-07 14:38:21 +0900 (f11320c) @@ -13,24 +13,25 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/input_adapter_options" +module Droonga + class Adapter + class PluginConfiguration + def initialize(adapter_class) + @adapter_class = adapter_class + end -class InputAdapterOptionsTest < Test::Unit::TestCase - def options(data) - Droonga::InputAdapterOptions.new(data) - end - - class PluginsTest < self - def plugins(data) - options(data).plugins - end + def name + configuration[:name] + end - def test_nothing - assert_equal([], plugins({})) - end + def name=(name) + configuration[:name] = name + end - def test_have_values - assert_equal(["groonga"], plugins("plugins" => ["groonga"])) + private + def configuration + @adapter_class.options[:plugin] ||= {} + end end end end Modified: lib/droonga/adapter_runner.rb (+79 -9) =================================================================== --- lib/droonga/adapter_runner.rb 2014-02-07 13:59:19 +0900 (ad24cbf) +++ lib/droonga/adapter_runner.rb 2014-02-07 14:38:21 +0900 (ce4493e) @@ -13,28 +13,98 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/input_adapter" -require "droonga/output_adapter" +require "droonga/message_matcher" +require "droonga/input_message" +require "droonga/output_message" +require "droonga/adapter" module Droonga class AdapterRunner - def initialize(dispatcher, input_adapter_options, output_adapter_options) + def initialize(dispatcher, plugins) @dispatcher = dispatcher - @input_adapter = InputAdapter.new(self, input_adapter_options) - @output_adapter = OutputAdapter.new(self, output_adapter_options) + default_plugins = ["error"] + plugins += (default_plugins - plugins) + @adapter_classes = collect_adapter_classes(plugins) end def shutdown - @input_adapter.shutdown - @output_adapter.shutdown end def adapt_input(message) - @input_adapter.adapt(message) + $log.trace("#{log_tag}: adapt_input: start", + :dataset => message["dataset"], + :type => message["type"]) + adapted_message = message + adapted_message["appliedAdapters"] = [] + @adapter_classes.each do |adapter_class| + adapter_class_id = adapter_class.id + pattern = adapter_class.message.input_pattern + if pattern + matcher = MessageMatcher.new(pattern) + $log.trace("#{log_tag}: adapt_input: skip: #{adapter_class_id}", + :pattern => pattern) + next unless matcher.match?(adapted_message) + end + $log.trace("#{log_tag}: adapt_input: use: #{adapter_class_id}") + input_message = InputMessage.new(adapted_message) + adapter = adapter_class.new + adapter.adapt_input(input_message) + adapted_message = input_message.adapted_message + adapted_message["appliedAdapters"] << adapter_class_id + end + $log.trace("#{log_tag}: adapt_input: done", + :dataset => adapted_message["dataset"], + :type => adapted_message["type"]) + adapted_message end def adapt_output(message) - @output_adapter.adapt(message) + $log.trace("#{log_tag}: adapt_output: start", + :dataset => message["dataset"], + :type => message["type"]) + adapted_message = message + applied_adapters = adapted_message["appliedAdapters"] + @adapter_classes.reverse_each do |adapter_class| + adapter_class_id = adapter_class.id + if applied_adapters + $log.trace("#{log_tag}: adapt_output: skip: #{adapter_class_id}: " + + "input adapter wasn't applied", + :applied_adapters => applied_adapters) + next unless applied_adapters.include?(adapter_class.id) + end + pattern = adapter_class.message.output_pattern + if pattern + matcher = MessageMatcher.new(pattern) + $log.trace("#{log_tag}: adapt_output: skip: #{adapter_class_id}", + :pattern => pattern) + next unless matcher.match?(adapted_message) + end + $log.trace("#{log_tag}: adapt_output: use: #{adapter_class_id}") + output_message = OutputMessage.new(adapted_message) + adapter = adapter_class.new + adapter.adapt_output(output_message) + adapted_message = output_message.adapted_message + end + $log.trace("#{log_tag}: adapt_output: done", + :dataset => adapted_message["dataset"], + :type => adapted_message["type"]) + adapted_message + end + + private + def collect_adapter_classes(plugins) + adapter_classes = [] + plugins.each do |plugin_name| + Adapter.adapter_classes.each do |adapter_class| + next unless adapter_class.plugin.name == plugin_name + adapter_classes << adapter_class + end + end + adapter_classes + end + + def log_tag + "adapter-runner" end end end Modified: lib/droonga/catalog/base.rb (+5 -11) =================================================================== --- lib/droonga/catalog/base.rb 2014-02-07 13:59:19 +0900 (a8ad9ae) +++ lib/droonga/catalog/base.rb 2014-02-07 14:38:21 +0900 (f3c753e) @@ -16,8 +16,6 @@ require "digest/sha1" require "zlib" require "droonga/message_processing_error" -require "droonga/input_adapter_options" -require "droonga/output_adapter_options" require "droonga/collector_options" require "droonga/planner_options" @@ -119,8 +117,12 @@ module Droonga return continuum[max][1] end + def datasets + @data["datasets"] || {} + end + def dataset(name) - dataset = @data["datasets"][name] + dataset = datasets[name] raise UnknownDataset.new(name) unless dataset dataset end @@ -139,14 +141,6 @@ module Droonga end end - def input_adapter_options - InputAdapterOptions.new(@data["input_adapter"]) - end - - def output_adapter_options - OutputAdapterOptions.new(@data["output_adapter"]) - end - def collector_options CollectorOptions.new(@data["collector"]) end Modified: lib/droonga/catalog/version1.rb (+0 -16) =================================================================== --- lib/droonga/catalog/version1.rb 2014-02-07 13:59:19 +0900 (53df3e8) +++ lib/droonga/catalog/version1.rb 2014-02-07 14:38:21 +0900 (9185842) @@ -18,22 +18,6 @@ require "droonga/catalog/base" module Droonga module Catalog class Version1 < Base - def initialize(*args) - super - normalize_input_adapter - normalize_output_adapter - end - - private - def normalize_input_adapter - @data["input_adapter"] ||= {} - @data["input_adapter"]["plugins"] ||= @options["plugins"] - end - - def normalize_output_adapter - @data["output_adapter"] ||= {} - @data["output_adapter"]["plugins"] ||= @options["plugins"] - end end end end Modified: lib/droonga/dispatcher.rb (+23 -6) =================================================================== --- lib/droonga/dispatcher.rb 2014-02-07 13:59:19 +0900 (de1529c) +++ lib/droonga/dispatcher.rb 2014-02-07 14:38:21 +0900 (00d1041) @@ -52,9 +52,7 @@ module Droonga @sessions = {} @current_id = 0 @local = Regexp.new("^#{@name}") - @adapter_runner = AdapterRunner.new(self, - Droonga.catalog.input_adapter_options, - Droonga.catalog.output_adapter_options) + @adapter_runners = create_adapter_runners @farm = Farm.new(name, @loop, :dispatcher => self) @forwarder = Forwarder.new(@loop) @replier = Replier.new(@forwarder) @@ -74,7 +72,9 @@ module Droonga @forwarder.shutdown @planner.shutdown @collector.shutdown - @adapter_runner.shutdown + @adapter_runners.each_value do |adapter_runner| + adapter_runner.shutdown + end @farm.shutdown @loop.stop @loop_thread.join @@ -119,7 +119,11 @@ module Droonga # # @see Replier#reply def reply(message) - adapted_message = @adapter_runner.adapt_output(@message.merge(message)) + adapted_message =****@messa*****(message) + adapter_runner = @adapter_runners[adapted_message["dataset"]] + if adapter_runner + adapted_message = adapter_runner.adapt_output(adapted_message) + end return if adapted_message["replyTo"].nil? @replier.reply(adapted_message) end @@ -212,7 +216,12 @@ module Droonga end def process_input_message(message) - adapted_message = @adapter_runner.adapt_input(message) + adapter_runner = @adapter_runners[message["dataset"]] + if adapter_runner + adapted_message = adapter_runner.adapt_input(message) + else + adapted_message = message + end plan =****@plann*****(adapted_message["type"], adapted_message) distributor = Distributor.new(self) distributor.distribute(plan) @@ -224,6 +233,14 @@ module Droonga raise MissingDatasetParameter.new unles****@messa*****?("dataset") end + def create_adapter_runners + runners = {} + Droonga.catalog.datasets.each do |name, configuration| + runners[name] = AdapterRunner.new(self, configuration["plugins"] || []) + end + runners + end + def log_tag "[#{Process.ppid}][#{Process.pid}] dispatcher" end Deleted: lib/droonga/plugin/input_adapter/groonga/select.rb (+0 -63) 100644 =================================================================== --- lib/droonga/plugin/input_adapter/groonga/select.rb 2014-02-07 13:59:19 +0900 (98f7a43) +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (C) 2013 Droonga Project -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 2.1 as published by the Free Software Foundation. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -module Droonga - class GroongaInputAdapter - class Select - def convert(select_request) - table = select_request["table"] - result_name = table + "_result" - match_columns = select_request["match_columns"] - match_to = match_columns ? match_columns.split(/ *\|\| */) : [] - query = select_request["query"] - output_columns = select_request["output_columns"] || "" - attributes = output_columns.split(/, */) - offset = (select_request["offset"] || "0").to_i - limit = (select_request["limit"] || "10").to_i - - search_request = { - "queries" => { - result_name => { - "source" => table, - "output" => { - "elements" => [ - "startTime", - "elapsedTime", - "count", - "attributes", - "records", - ], - "attributes" => attributes, - "offset" => offset, - "limit" => limit, - }, - } - } - } - if query - condition = { - "query" => query, - "matchTo"=> match_to, - "defaultOperator"=> "&&", - "allowPragma"=> false, - "allowColumn"=> true, - } - search_request["queries"][result_name]["condition"] = condition - end - search_request - end - end - end -end Deleted: lib/droonga/plugin/output_adapter/groonga.rb (+0 -44) 100644 =================================================================== --- lib/droonga/plugin/output_adapter/groonga.rb 2014-02-07 13:59:19 +0900 (c260bdd) +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (C) 2013-2014 Droonga Project -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 2.1 as published by the Free Software Foundation. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -require "droonga/output_adapter_plugin" - -module Droonga - class GroongaOutputAdapter < Droonga::OutputAdapterPlugin - repository.register("groonga", self) - - command :convert_select, - :pattern => ["originalTypes", :include?, "select"] - def convert_select(output_message) - command = Select.new - output_message.body = command.convert(output_message.body) - end - - groonga_results = [ - "table_create.result", - "table_remove.result", - "column_create.result", - ] - command :convert_generic_result, - :pattern => ["replyTo.type", :in, groonga_results] - def convert_generic_result(output_message) - if output_message.body.include?("result") - output_message.body = output_message.body["result"] - end - end - end -end - -require "droonga/plugin/output_adapter/groonga/select" Deleted: lib/droonga/plugin/output_adapter/groonga/select.rb (+0 -54) 100644 =================================================================== --- lib/droonga/plugin/output_adapter/groonga/select.rb 2014-02-07 13:59:19 +0900 (35917fa) +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (C) 2013 Droonga Project -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 2.1 as published by the Free Software Foundation. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -module Droonga - class GroongaOutputAdapter - class Select - def convert(search_response) - select_responses = search_response.collect do |key, value| - status_code = 0 - - start_time = value["startTime"] - start_time_in_unix_time = if start_time - Time.parse(start_time).to_f - else - Time.now.to_f - end - elapsed_time = value["elapsedTime"] || 0 - count = value["count"] - - attributes = value["attributes"] || [] - converted_attributes = attributes.collect do |attribute| - name = attribute["name"] - type = attribute["type"] - [name, type] - end - - header = [status_code, start_time_in_unix_time, elapsed_time] - records = value["records"] - if records.empty? - results = [[count], converted_attributes] - else - results = [[count], converted_attributes, records] - end - body = [results] - - [header, body] - end - select_responses.first - end - end - end -end Modified: lib/droonga/plugin_loader.rb (+8 -0) =================================================================== --- lib/droonga/plugin_loader.rb 2014-02-07 13:59:19 +0900 (28d5463) +++ lib/droonga/plugin_loader.rb 2014-02-07 14:38:21 +0900 (c19cbe0) @@ -15,6 +15,8 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +require "pathname" + module Droonga class PluginLoader class << self @@ -29,6 +31,12 @@ module Droonga loader.load end end + Pathname.glob("#{load_path}/droonga/plugins/*.rb") do |plugin_path| + relative_plugin_path = + plugin_path.relative_path_from(Pathname(load_path)) + require_path = relative_plugin_path.to_s.gsub(/\.rb\z/, "") + require require_path + end end end end Copied: lib/droonga/plugins/crud.rb (+12 -10) 62% =================================================================== --- lib/droonga/plugin/output_adapter/crud.rb 2014-02-07 13:59:19 +0900 (0a29886) +++ lib/droonga/plugins/crud.rb 2014-02-07 14:38:21 +0900 (19ec412) @@ -13,19 +13,21 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/output_adapter_plugin" +require "droonga/adapter" module Droonga - class CRUDOutputAdapter < Droonga::OutputAdapterPlugin - repository.register("crud", self) + module Plugins + module CRUD + class Adapter < Droonga::Adapter + plugin.name = "crud" + message.input_pattern = ["type", :equal, "add"] + message.output_pattern = ["body.success", :exist?] - command :convert_success, - :pattern => ["replyTo.type", :equal, "add.result"] - def convert_success(output_message) - if output_message.body.include?("success") - success = output_message.body["success"] - unless success.nil? - output_message.body = output_message.body["success"] + def adapt_output(output_message) + success = output_message.body["success"] + unless success.nil? + output_message.body = output_message.body["success"] + end end end end Added: lib/droonga/plugins/error.rb (+49 -0) 100644 =================================================================== --- /dev/null +++ lib/droonga/plugins/error.rb 2014-02-07 14:38:21 +0900 (e9719f6) @@ -0,0 +1,49 @@ +# Copyright (C) 2014 Droonga Project +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +require "droonga/adapter" + +module Droonga + module Plugins + module Error + class Adapter < Droonga::Adapter + plugin.name = "error" + message.output_pattern = ["body.errors", :exist?] + + def adapt_output(output_message) + errors = output_message.body["errors"] + if errors && !errors.empty? + output_message.errors = errors + + status_codes = [] + errors.values.each do |error| + status_codes << error["statusCode"] + end + status_codes = status_codes.uniq + if status_codes.size == 1 + output_message.status_code = status_codes.first + else + output_message.status_code = MessageProcessingError::STATUS_CODE + end + + output_message.body = errors.values.first["body"] + else + output_message.body.delete("errors") + end + end + end + end + end +end Renamed: lib/droonga/plugins/groonga.rb (+2 -18) 59% =================================================================== --- lib/droonga/plugin/input_adapter/groonga.rb 2014-02-07 13:59:19 +0900 (4996e0b) +++ lib/droonga/plugins/groonga.rb 2014-02-07 14:38:21 +0900 (2a9f470) @@ -13,21 +13,5 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/input_adapter_plugin" - -module Droonga - class GroongaInputAdapter < Droonga::InputAdapterPlugin - repository.register("groonga", self) - - command :select - def select(input_message) - command = Select.new - select_request = input_message.body - search_request = command.convert(select_request) - input_message.command = "search" - input_message.body = search_request - end - end -end - -require "droonga/plugin/input_adapter/groonga/select" +require "droonga/plugins/groonga/generic" +require "droonga/plugins/groonga/select" Renamed: lib/droonga/plugins/groonga/generic.rb (+17 -10) 58% =================================================================== --- lib/droonga/plugin/output_adapter/crud.rb 2014-02-07 13:59:19 +0900 (0a29886) +++ lib/droonga/plugins/groonga/generic.rb 2014-02-07 14:38:21 +0900 (eba9bd4) @@ -13,19 +13,26 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/output_adapter_plugin" +require "droonga/adapter" module Droonga - class CRUDOutputAdapter < Droonga::OutputAdapterPlugin - repository.register("crud", self) + module Plugins + module Groonga + module Generic + class Adapter < Droonga::Adapter + plugin.name = "groonga" - command :convert_success, - :pattern => ["replyTo.type", :equal, "add.result"] - def convert_success(output_message) - if output_message.body.include?("success") - success = output_message.body["success"] - unless success.nil? - output_message.body = output_message.body["success"] + groonga_commands = [ + "table_create", + "table_remove", + "column_create", + ] + message.input_pattern = ["type", :in, groonga_commands] + message.output_pattern = ["body.result", :exist?] + + def adapt_output(output_message) + output_message.body = output_message.body["result"] + end end end end Added: lib/droonga/plugins/groonga/select.rb (+125 -0) 100644 =================================================================== --- /dev/null +++ lib/droonga/plugins/groonga/select.rb 2014-02-07 14:38:21 +0900 (55543c0) @@ -0,0 +1,125 @@ +# Copyright (C) 2013-2014 Droonga Project +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +require "droonga/adapter" + +module Droonga + module Plugins + module Groonga + module Select + class RequestConverter + def convert(select_request) + table = select_request["table"] + result_name = table + "_result" + match_columns = select_request["match_columns"] + match_to = match_columns ? match_columns.split(/ *\|\| */) : [] + query = select_request["query"] + output_columns = select_request["output_columns"] || "" + attributes = output_columns.split(/, */) + offset = (select_request["offset"] || "0").to_i + limit = (select_request["limit"] || "10").to_i + + search_request = { + "queries" => { + result_name => { + "source" => table, + "output" => { + "elements" => [ + "startTime", + "elapsedTime", + "count", + "attributes", + "records", + ], + "attributes" => attributes, + "offset" => offset, + "limit" => limit, + }, + } + } + } + if query + condition = { + "query" => query, + "matchTo"=> match_to, + "defaultOperator"=> "&&", + "allowPragma"=> false, + "allowColumn"=> true, + } + search_request["queries"][result_name]["condition"] = condition + end + search_request + end + end + + class ResponseConverter + def convert(search_response) + select_responses = search_response.collect do |key, value| + status_code = 0 + + start_time = value["startTime"] + start_time_in_unix_time = if start_time + Time.parse(start_time).to_f + else + Time.now.to_f + end + elapsed_time = value["elapsedTime"] || 0 + count = value["count"] + + attributes = value["attributes"] || [] + converted_attributes = attributes.collect do |attribute| + name = attribute["name"] + type = attribute["type"] + [name, type] + end + + header = [status_code, start_time_in_unix_time, elapsed_time] + records = value["records"] + if records.empty? + results = [[count], converted_attributes] + else + results = [[count], converted_attributes, records] + end + body = [results] + + [header, body] + end + select_responses.first + end + end + + class Adapter < Droonga::Adapter + plugin.name = "groonga" + message.input_pattern = ["type", :equal, "select"] + + def adapt_input(input_message) + converter = RequestConverter.new + select_request = input_message.body + search_request = converter.convert(select_request) + input_message.command = "search" + input_message.body = search_request + end + + def adapt_output(output_message) + converter = ResponseConverter.new + search_response = output_message.body + select_response = converter.convert(search_response) + output_message.body = select_response + end + end + end + end + end +end Modified: sample/cluster/catalog.json (+1 -6) =================================================================== --- sample/cluster/catalog.json 2014-02-07 13:59:19 +0900 (098a0b1) +++ sample/cluster/catalog.json 2014-02-07 14:38:21 +0900 (902c12b) @@ -11,6 +11,7 @@ "datasets": { "Droonga": { "workers": 2, + "plugins": ["crud", "groonga"], "handler": { "plugins": ["search", "groonga", "add"] }, @@ -40,12 +41,6 @@ } } }, - "input_adapter": { - "plugins": ["groonga"] - }, - "output_adapter": { - "plugins": ["crud", "groonga"] - }, "collector": { "plugins": ["basic", "search"] }, Modified: test/command/config/default/catalog.json (+1 -6) =================================================================== --- test/command/config/default/catalog.json 2014-02-07 13:59:19 +0900 (60988d8) +++ test/command/config/default/catalog.json 2014-02-07 14:38:21 +0900 (beeaee4) @@ -13,6 +13,7 @@ "handler": { "plugins": ["search", "groonga", "add"] }, + "plugins": ["groonga", "crud"], "number_of_replicas": 2, "number_of_partitions": 2, "partition_key": "_key", @@ -59,12 +60,6 @@ } } }, - "input_adapter": { - "plugins": ["groonga"] - }, - "output_adapter": { - "plugins": ["crud", "groonga"] - }, "collector": { "plugins": ["basic", "search"] }, Modified: test/unit/catalog/test_version1.rb (+0 -94) =================================================================== --- test/unit/catalog/test_version1.rb 2014-02-07 13:59:19 +0900 (2e649fa) +++ test/unit/catalog/test_version1.rb 2014-02-07 14:38:21 +0900 (850f460) @@ -174,100 +174,6 @@ class CatalogTest < Test::Unit::TestCase end end - class InputAdapterOptionsTest < self - def options(data) - catalog = create_catalog(minimum_data.merge(data), "base-path") - catalog.input_adapter_options - end - - class PluginsTest < self - def plugins(data) - options(data).plugins - end - - def test_nothing - assert_equal([], plugins({})) - end - - def test_options - data = { - "options" => { - "plugins" => ["groonga"], - } - } - assert_equal(["groonga"], plugins(data)) - end - - def test_input_adapter - data = { - "input_adapter" => { - "plugins" => ["groonga"], - } - } - assert_equal(["groonga"], plugins(data)) - end - - def test_options_and_input_adapter - data = { - "options" => { - "plugins" => ["basic"], - }, - "input_adapter" => { - "plugins" => ["groonga"], - } - } - assert_equal(["groonga"], plugins(data)) - end - end - end - - class OutputAdapterOptionsTest < self - def options(data) - catalog = create_catalog(minimum_data.merge(data), "base-path") - catalog.output_adapter_options - end - - class PluginsTest < self - def plugins(data) - options(data).plugins - end - - def test_nothing - assert_equal([], plugins({})) - end - - def test_options - data = { - "options" => { - "plugins" => ["groonga"], - } - } - assert_equal(["groonga"], plugins(data)) - end - - def test_output_adapter - data = { - "output_adapter" => { - "plugins" => ["groonga"], - } - } - assert_equal(["groonga"], plugins(data)) - end - - def test_options_and_output_adapter - data = { - "options" => { - "plugins" => ["basic"], - }, - "output_adapter" => { - "plugins" => ["groonga"], - } - } - assert_equal(["groonga"], plugins(data)) - end - end - end - class CollectorOptionsTest < self def options(data) catalog = create_catalog(minimum_data.merge(data), "base-path") Deleted: test/unit/plugin/input_adapter/groonga/test_select.rb (+0 -248) 100644 =================================================================== --- test/unit/plugin/input_adapter/groonga/test_select.rb 2014-02-07 13:59:19 +0900 (10308ca) +++ /dev/null @@ -1,248 +0,0 @@ - -# Copyright (C) 2013 Droonga Project -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 2.1 as published by the Free Software Foundation. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -require "droonga/plugin/input_adapter/groonga/select" - -class InputAdapterGroongaSelectTest < Test::Unit::TestCase - def setup - @select = Droonga::GroongaInputAdapter::Select.new - end - - class RequestTest < self - def test_empty - select_request = { - "table" => "EmptyTable", - "output_columns" => "_id", - } - - expected_search_request = { - "queries" => { - "EmptyTable_result" => { - "source" => "EmptyTable", - "output" => { - "elements" => [ - "startTime", - "elapsedTime", - "count", - "attributes", - "records", - ], - "attributes" => ["_id"], - "offset" => 0, - "limit" => 10, - }, - }, - }, - } - - assert_equal(expected_search_request, convert(select_request)) - end - - private - def convert(select_request) - @select.convert(select_request) - end - - class OutputColumnsTest < self - def assert_attributes(expected_attributes, output_columns) - select_request = { - "table" => "EmptyTable", - "output_columns" => output_columns, - } - - expected_search_request = { - "queries" => { - "EmptyTable_result" => { - "source" => "EmptyTable", - "output" => { - "elements" => [ - "startTime", - "elapsedTime", - "count", - "attributes", - "records", - ], - "attributes" => expected_attributes, - "offset" => 0, - "limit" => 10, - }, - }, - }, - } - assert_equal(expected_search_request, convert(select_request)) - end - - def test_multiple_columns - assert_attributes(["_id", "_key"], "_id,_key") - end - - class FunctionTest < self - def test_single_argument - assert_attributes(["snippet_html(content)"], "snippet_html(content)") - end - - def test_with_columns - assert_attributes(["_id","_key","snippet_html(content)"], "_id,_key,snippet_html(content)") - end - end - end - - class MatchColumnsTest < self - def assert_matchTo(expected_matchTo, match_columns) - select_request = { - "table" => "EmptyTable", - "match_columns" => match_columns, - "query" => "QueryTest", - "output_columns" => "_id", - } - - expected_search_request = { - "queries" => { - "EmptyTable_result" => { - "source" => "EmptyTable", - "condition"=> { - "query" => "QueryTest", - "matchTo"=> expected_matchTo, - "defaultOperator"=> "&&", - "allowPragma"=> false, - "allowColumn"=> true, - }, - "output" => { - "elements" => [ - "startTime", - "elapsedTime", - "count", - "attributes", - "records", - ], - "attributes" => ["_id"], - "offset" => 0, - "limit" => 10, - }, - }, - }, - } - assert_equal(expected_search_request, convert(select_request)) - end - - def test_single_column - assert_matchTo(["_key"], "_key") - end - - def test_multiple_columns - assert_matchTo(["_key", "content"], "_key || content") - end - end - - class OffsetTest < self - def assert_offset(expected_offset, offset) - select_request = { - "table" => "EmptyTable", - "output_columns" => "_id", - } - select_request["offset"] = offset unless offset.nil? - - expected_search_request = { - "queries" => { - "EmptyTable_result" => { - "source" => "EmptyTable", - "output" => { - "elements" => [ - "startTime", - "elapsedTime", - "count", - "attributes", - "records", - ], - "attributes" => ["_id"], - "offset" => expected_offset, - "limit" => 10, - }, - }, - }, - } - assert_equal(expected_search_request, convert(select_request)) - end - - def test_zero - assert_offset(0, "0") - end - - def test_large - assert_offset(100, "100") - end - - def test_integer - assert_offset(100, 100) - end - - def test_default - assert_offset(0, nil) - end - end - - class LimitTest < self - def assert_limit(expected_limit, limit) - select_request = { - "table" => "EmptyTable", - "output_columns" => "_id", - } - select_request["limit"] = limit unless limit.nil? - - expected_search_request = { - "queries" => { - "EmptyTable_result" => { - "source" => "EmptyTable", - "output" => { - "elements" => [ - "startTime", - "elapsedTime", - "count", - "attributes", - "records", - ], - "attributes" => ["_id"], - "offset" => 0, - "limit" => expected_limit, - }, - }, - }, - } - assert_equal(expected_search_request, convert(select_request)) - end - - def test_zero - assert_limit(0, "0") - end - - def test_large - assert_limit(100, "100") - end - - def test_negative - assert_limit(-1, "-1") - end - - def test_integer - assert_limit(100, 100) - end - - def test_default - assert_limit(10, nil) - end - end - end -end Added: test/unit/plugins/groonga/select/test_adapter_input.rb (+213 -0) 100644 =================================================================== --- /dev/null +++ test/unit/plugins/groonga/select/test_adapter_input.rb 2014-02-07 14:38:21 +0900 (eb62663) @@ -0,0 +1,213 @@ +# Copyright (C) 2013-2014 Droonga Project +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +require "droonga/plugins/groonga/select" + +class GroongaSelectAdapterInputTest < Test::Unit::TestCase + private + def convert(select_request) + converter = Droonga::Plugins::Groonga::Select::RequestConverter.new + converter.convert(select_request) + end + + class OutputColumnsTest < self + def assert_attributes(expected_attributes, output_columns) + select_request = { + "table" => "EmptyTable", + "output_columns" => output_columns, + } + + expected_search_request = { + "queries" => { + "EmptyTable_result" => { + "source" => "EmptyTable", + "output" => { + "elements" => [ + "startTime", + "elapsedTime", + "count", + "attributes", + "records", + ], + "attributes" => expected_attributes, + "offset" => 0, + "limit" => 10, + }, + }, + }, + } + assert_equal(expected_search_request, convert(select_request)) + end + + def test_multiple_columns + assert_attributes(["_id", "_key"], "_id,_key") + end + + class FunctionTest < self + def test_single_argument + assert_attributes(["snippet_html(content)"], "snippet_html(content)") + end + + def test_with_columns + assert_attributes(["_id","_key","snippet_html(content)"], "_id,_key,snippet_html(content)") + end + end + end + + class MatchColumnsTest < self + def assert_matchTo(expected_matchTo, match_columns) + select_request = { + "table" => "EmptyTable", + "match_columns" => match_columns, + "query" => "QueryTest", + "output_columns" => "_id", + } + + expected_search_request = { + "queries" => { + "EmptyTable_result" => { + "source" => "EmptyTable", + "condition"=> { + "query" => "QueryTest", + "matchTo"=> expected_matchTo, + "defaultOperator"=> "&&", + "allowPragma"=> false, + "allowColumn"=> true, + }, + "output" => { + "elements" => [ + "startTime", + "elapsedTime", + "count", + "attributes", + "records", + ], + "attributes" => ["_id"], + "offset" => 0, + "limit" => 10, + }, + }, + }, + } + assert_equal(expected_search_request, convert(select_request)) + end + + def test_single_column + assert_matchTo(["_key"], "_key") + end + + def test_multiple_columns + assert_matchTo(["_key", "content"], "_key || content") + end + end + + class OffsetTest < self + def assert_offset(expected_offset, offset) + select_request = { + "table" => "EmptyTable", + "output_columns" => "_id", + } + select_request["offset"] = offset unless offset.nil? + + expected_search_request = { + "queries" => { + "EmptyTable_result" => { + "source" => "EmptyTable", + "output" => { + "elements" => [ + "startTime", + "elapsedTime", + "count", + "attributes", + "records", + ], + "attributes" => ["_id"], + "offset" => expected_offset, + "limit" => 10, + }, + }, + }, + } + assert_equal(expected_search_request, convert(select_request)) + end + + def test_zero + assert_offset(0, "0") + end + + def test_large + assert_offset(100, "100") + end + + def test_integer + assert_offset(100, 100) + end + + def test_default + assert_offset(0, nil) + end + end + + class LimitTest < self + def assert_limit(expected_limit, limit) + select_request = { + "table" => "EmptyTable", + "output_columns" => "_id", + } + select_request["limit"] = limit unless limit.nil? + + expected_search_request = { + "queries" => { + "EmptyTable_result" => { + "source" => "EmptyTable", + "output" => { + "elements" => [ + "startTime", + "elapsedTime", + "count", + "attributes", + "records", + ], + "attributes" => ["_id"], + "offset" => 0, + "limit" => expected_limit, + }, + }, + }, + } + assert_equal(expected_search_request, convert(select_request)) + end + + def test_zero + assert_limit(0, "0") + end + + def test_large + assert_limit(100, "100") + end + + def test_negative + assert_limit(-1, "-1") + end + + def test_integer + assert_limit(100, 100) + end + + def test_default + assert_limit(10, nil) + end + end +end Renamed: test/unit/plugins/groonga/select/test_adapter_output.rb (+12 -13) 73% =================================================================== --- test/unit/plugin/output_adapter/groonga/test_select.rb 2014-02-07 13:59:19 +0900 (36efbcd) +++ test/unit/plugins/groonga/select/test_adapter_output.rb 2014-02-07 14:38:21 +0900 (ddd57e1) @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Droonga Project +# Copyright (C) 2013-2014 Droonga Project # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,14 +13,16 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/plugin/output_adapter/groonga/select" +require "droonga/plugins/groonga/select" -class OutputAdapterGroongaSelectTest < Test::Unit::TestCase - def setup - @select = Droonga::GroongaOutputAdapter::Select.new +class GroongaSelectAdapterOutputTest < Test::Unit::TestCase + private + def convert(search_response) + converter = Droonga::Plugins::Groonga::Select::ResponseConverter.new + converter.convert(search_response) end - class ResponseTest < self + class RecordsTest < self def test_empty start_time = "2001-08-02T10:45:23.5+09:00" elapsed_time = 0 @@ -41,15 +43,12 @@ class OutputAdapterGroongaSelectTest < Test::Unit::TestCase status_code = 0 start_time_in_unix_time = Time.parse(start_time).to_f headers = [["_id","UInt32"]] - expected_select_response = [[status_code, start_time_in_unix_time, elapsed_time], - [[[count], headers]]] + expected_select_response = [ + [status_code, start_time_in_unix_time, elapsed_time], + [[[count], headers]], + ] assert_equal(expected_select_response, convert(search_response)) end - - private - def convert(search_response) - @select.convert(search_response) - end end end -------------- next part -------------- HTML����������������������������...Download