Kouhei Sutou
null+****@clear*****
Mon Feb 17 17:28:19 JST 2014
Kouhei Sutou 2014-02-17 17:28:19 +0900 (Mon, 17 Feb 2014) New Revision: ed932648b0ec9b5a947c1fcf9ba7dc95ac5a4ec8 https://github.com/droonga/fluent-plugin-droonga/commit/ed932648b0ec9b5a947c1fcf9ba7dc95ac5a4ec8 Message: collector: use new plugin style Added files: lib/droonga/collector_runner.rb lib/droonga/plugins/basic.rb Removed files: lib/droonga/collector_plugin.rb lib/droonga/legacy_pluggable.rb lib/droonga/legacy_plugin.rb lib/droonga/plugin/collector/search.rb lib/droonga/plugin_registerable.rb test/unit/test_legacy_plugin.rb test/unit/test_legacy_plugin_repository.rb Modified files: lib/droonga/collector.rb lib/droonga/dispatcher.rb lib/droonga/plugins/search.rb lib/droonga/session.rb Renamed files: lib/droonga/collector_message.rb (from lib/droonga/legacy_plugin_repository.rb) lib/droonga/plugin/metadata/collector_message.rb (from test/unit/test_command_repository.rb) lib/droonga/reducer.rb (from lib/droonga/plugin/collector/basic.rb) test/unit/plugins/search/test_collector.rb (from test/unit/plugin/collector/test_search.rb) test/unit/plugins/test_basic.rb (from test/unit/plugin/collector/test_basic.rb) Modified: lib/droonga/collector.rb (+10 -10) =================================================================== --- lib/droonga/collector.rb 2014-02-17 15:39:00 +0900 (d7bcb24) +++ lib/droonga/collector.rb 2014-02-17 17:28:19 +0900 (bb94d65) @@ -13,24 +13,24 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/legacy_pluggable" -require "droonga/collector_plugin" +require "droonga/pluggable" +require "droonga/plugin/metadata/collector_message" module Droonga class Collector - include LegacyPluggable + extend Pluggable - def initialize(plugins) - load_plugins(plugins) + class << self + def message + Plugin::Metadata::CollectorMessage.new(self) + end end - private - def instantiate_plugin(name) - CollectorPlugin.repository.instantiate(name) + def initialize end - def log_tag - "collector" + def collect(message) + raise NotImplemented, "#{self.class.name}\##{__method__} must implement." end end end Renamed: lib/droonga/collector_message.rb (+41 -24) 51% =================================================================== --- lib/droonga/legacy_plugin_repository.rb 2014-02-17 15:39:00 +0900 (2de3124) +++ lib/droonga/collector_message.rb 2014-02-17 17:28:19 +0900 (cdc4e1c) @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Droonga Project +# Copyright (C) 2014 Droonga Project # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -14,41 +14,58 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA module Droonga - class LegacyPluginRepository - include Enumerable + class CollectorMessage + attr_reader :raw + def initialize(raw) + @raw = raw + end - def initialize - @plugins = {} + def valid? + task and step and values end - def each(&block) - @plugins.each(&block) + def [](key) + @raw[key] end - def register(name, klass) - @plugins[name] = klass + def task + @raw["task"] end - def [](name) - @plugins[name] + def step + task["step"] end - def clear - @plugins.clear + def type + step["type"] end - def instantiate(name, *args, &block) - plugin_class = self[name] - if plugin_class.nil? - # TODO: use the original error - raise ArgumentError, "unknown plugin: <#{name}>" - end - begin - plugin_class.new(*args, &block) - rescue - p [plugin_class, plugin_class.method(:new), plugin_class.method(:new).arity, args.size] - raise + def values + task["values"] + end + + def body + step["body"] + end + + def input + if body + body[name] + else + nil end end + + def outputs + step["outputs"] + end + + def name + @raw["name"] + end + + def value + @raw["value"] + end end end Deleted: lib/droonga/collector_plugin.rb (+0 -50) 100644 =================================================================== --- lib/droonga/collector_plugin.rb 2014-02-17 15:39:00 +0900 (b94db6d) +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2013 Droonga Project -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 2.1 as published by the Free Software Foundation. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -require "droonga/legacy_plugin" - -module Droonga - class CollectorPlugin < LegacyPlugin - extend PluginRegisterable - - attr_reader :task, :input_name, :step, :output_values, :body, :output_names - def initialize - super() - end - - def process(command, message) - return false unless message.is_a? Hash - @task = message["task"] - return false unles****@task*****_a? Hash - @step = @task["step"] - return false unles****@step*****_a? Hash - @output_values = @task["values"] - @body = @step["body"] - @output_names = @step["outputs"] - @id = message["id"] - @value = message["value"] - @input_name = message["name"] - super(command, @value) - true - end - - # TODO: consider better name - def emit(name, value) - @output_values[name] = value - end - end -end Added: lib/droonga/collector_runner.rb (+61 -0) 100644 =================================================================== --- /dev/null +++ lib/droonga/collector_runner.rb 2014-02-17 17:28:19 +0900 (76a15d1) @@ -0,0 +1,61 @@ +# Copyright (C) 2014 Droonga Project +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +require "droonga/message_matcher" +require "droonga/collector" +require "droonga/collector_message" + +module Droonga + class CollectorRunner + def initialize(plugins) + default_plugins = ["basic"] + plugins += (default_plugins - plugins) + @collector_classes = Collector.find_sub_classes(plugins) + end + + def shutdown + end + + def collect(message) + collector_message = CollectorMessage.new(message) + $log.trace("#{log_tag}: collect: start", + :type => collector_message.type) + collector_class = find_collector_class(message) + if collector_class.nil? + raise UnsupportedMessageError.new(:collector, message) + end + collector = collector_class.new + collector.collect(collector_message) + $log.trace("#{log_tag}: collector: done") + end + + private + def find_collector_class(message) + @collector_classes.find do |collector_class| + pattern = collector_class.message.pattern + if pattern + matcher = MessageMatcher.new(pattern) + matcher.match?(message) + else + false + end + end + end + + def log_tag + "collector-runner" + end + end +end Modified: lib/droonga/dispatcher.rb (+32 -13) =================================================================== --- lib/droonga/dispatcher.rb 2014-02-17 15:39:00 +0900 (4e25495) +++ lib/droonga/dispatcher.rb 2014-02-17 17:28:19 +0900 (acd69d1) @@ -18,7 +18,7 @@ require "tsort" require "droonga/adapter_runner" require "droonga/planner_runner" -require "droonga/collector" +require "droonga/collector_runner" require "droonga/farm" require "droonga/session" require "droonga/replier" @@ -56,13 +56,12 @@ module Droonga @sessions = {} @current_id = 0 @local = Regexp.new("^#{@name}") - @adapter_runners = create_runners(AdapterRunner) + @adapter_runners = create_adapter_runners @farm = Farm.new(name, @catalog, @loop, :dispatcher => self) @forwarder = Forwarder.new(@loop) @replier = Replier.new(@forwarder) - @planner_runners = create_runners(PlannerRunner) - # TODO: make customizable - @collector = Collector.new(["basic", "search"]) + @planner_runners = create_planner_runners + @collector_runners = create_collector_runners end def start @@ -78,7 +77,9 @@ module Droonga @planner_runners.each_value do |planner_runner| planner_runner.shutdown end - @collector.shutdown + @collector_runners.each_value do |collector_runner| + collector_runner.shutdown + end @adapter_runners.each_value do |adapter_runner| adapter_runner.shutdown end @@ -145,7 +146,9 @@ module Droonga steps = message["steps"] if steps session_planner = SessionPlanner.new(self, steps) - session = session_planner.create_session(id, @collector) + dataset = message["dataset"] || @message["dataset"] + collector_runner = @collector_runners[dataset] + session = session_planner.create_session(id, collector_runner) @sessions[id] = session else #todo: take cases receiving result before its query into account @@ -235,8 +238,6 @@ module Droonga target_message = error.message raise UnknownCommand.new(target_message["type"], target_message["dataset"]) - rescue Droonga::LegacyPluggable::UnknownPlugin => error - raise UnknownCommand.new(error.command, message["dataset"]) end def assert_valid_message(message) @@ -249,14 +250,32 @@ module Droonga end end - def create_runners(runner_class) + def create_runners runners = {} @catalog.datasets.each do |name, configuration| - runners[name] = runner_class.new(self, configuration["plugins"] || []) + runners[name] = yield(configuration) end runners end + def create_adapter_runners + create_runners do |configuration| + AdapterRunner.new(self, configuration["plugins"] || []) + end + end + + def create_planner_runners + create_runners do |configuration| + PlannerRunner.new(self, configuration["plugins"] || []) + end + end + + def create_collector_runners + create_runners do |configuration| + CollectorRunner.new(configuration["plugins"] || []) + end + end + def log_tag "[#{Process.ppid}][#{Process.pid}] dispatcher" end @@ -269,7 +288,7 @@ module Droonga @steps = steps end - def create_session(id, collector) + def create_session(id, collector_runner) resolve_descendants tasks = [] inputs = {} @@ -289,7 +308,7 @@ module Droonga end end end - Session.new(id, @dispatcher, collector, tasks, inputs) + Session.new(id, @dispatcher, collector_runner, tasks, inputs) end def resolve_descendants Deleted: lib/droonga/legacy_pluggable.rb (+0 -66) 100644 =================================================================== --- lib/droonga/legacy_pluggable.rb 2014-02-17 15:39:00 +0900 (acf4fb9) +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (C) 2013-2014 Droonga Project -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 2.1 as published by the Free Software Foundation. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -module Droonga - module LegacyPluggable - class UnknownPlugin < StandardError - attr_reader :command - - def initialize(command) - @command = command - end - end - - def shutdown - $log.trace("#{log_tag}: shutdown: plugin: start") - @plugins.each do |plugin| - plugin.shutdown - end - $log.trace("#{log_tag}: shutdown: plugin: done") - end - - def processable?(command) - not find_plugin(command).nil? - end - - def process(command, *arguments) - plugin = find_plugin(command) - $log.trace("#{log_tag}: process: start: <#{command}>", - :plugin => plugin.class) - raise UnknownPlugin.new(command) if plugin.nil? - result = plugin.process(command, *arguments) - $log.trace("#{log_tag}: process: done: <#{command}>", - :plugin => plugin.class) - result - end - - private - def load_plugins(names) - @plugins = names.collect do |name| - plugin = instantiate_plugin(name) - if plugin.nil? - raise "unknown plugin: <#{name}>: TODO: improve error handling" - end - plugin - end - end - - def find_plugin(command) - @plugins.find do |plugin| - plugin.processable?(command) - end - end - end -end Deleted: lib/droonga/legacy_plugin.rb (+0 -57) 100644 =================================================================== --- lib/droonga/legacy_plugin.rb 2014-02-17 15:39:00 +0900 (f34ad60) +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2013 Droonga Project -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 2.1 as published by the Free Software Foundation. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -require "droonga/plugin_registerable" - -module Droonga - class LegacyPlugin - def initialize - end - - def start - end - - def shutdown - end - - def processable?(command) - self.class.processable?(command) - end - - def process(command, *arguments) - run_command(command, *arguments) - rescue => exception - process_error(command, exception, arguments) - end - - private - def run_command(command, *arguments) - if command.is_a?(Command) - method_name = command.method_name - else - method_name = self.class.method_name(command) - end - __send__(method_name, *arguments) - end - - def process_error(command, error, arguments) - Logger.error("error while processing: <#{command}>", - error, - :arguments => arguments) - end - end -end Deleted: lib/droonga/plugin/collector/search.rb (+0 -98) 100644 =================================================================== --- lib/droonga/plugin/collector/search.rb 2014-02-17 15:39:00 +0900 (8adfb74) +++ /dev/null @@ -1,98 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2013-2014 Droonga Project -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 2.1 as published by the Free Software Foundation. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -require "droonga/plugin/collector/basic" - -module Droonga - class SearchCollector < BasicCollector - repository.register("search", self) - - command :search_gather - def search_gather(result) - output = body ? body[input_name] : input_name - if output.is_a?(Hash) - elements = output["elements"] - if elements and elements.is_a?(Hash) - # because "count" mapper requires all records, - # I have to apply it at first, before "limit" and "offset" are applied. - count_mapper = elements["count"] - if count_mapper - if count_mapper["no_output"] - result.delete("count") - else - result["count"] = result[count_mapper["target"]].size - end - end - - records_mapper = elements["records"] - if records_mapper and result["records"] - if records_mapper["no_output"] - result.delete("records") - else - result["records"] = apply_output_range(result["records"], records_mapper) - result["records"] = apply_output_attributes_and_format(result["records"], records_mapper) - end - end - end - output = output["output"] - end - emit(output, result) - end - - def apply_output_attributes_and_format(items, output) - attributes = output["attributes"] || [] - if output["format"] == "complex" - items.collect! do |item| - complex_item = {} - attributes.each_with_index do |label, index| - complex_item[label] = item[index] - end - complex_item - end - else - items.collect! do |item| - item[0...attributes.size] - end - end - items - end - - command :search_reduce - def search_reduce(request) - #XXX This is just a workaround. Errors should be handled by the framework itself. - if input_name == "errors" - return reduce(request) - end - - return unless request - body[input_name].each do |output, elements| - value = request - old_value = output_values[output] - value = reduce_elements(elements, old_value, request) if old_value - emit(output, value) - end - end - - def reduce_elements(elements, left_values, right_values) - result = {} - elements.each do |key, deal| - result[key] = reduce_value(deal, left_values[key], right_values[key]) - end - result - end - end -end Renamed: lib/droonga/plugin/metadata/collector_message.rb (+19 -19) 53% =================================================================== --- test/unit/test_command_repository.rb 2014-02-17 15:39:00 +0900 (db3d7d8) +++ lib/droonga/plugin/metadata/collector_message.rb 2014-02-17 17:28:19 +0900 (adb38bb) @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Droonga Project +# Copyright (C) 2014 Droonga Project # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,27 +13,27 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/command_repository" +module Droonga + module Plugin + module Metadata + class CollectorMessage + def initialize(plugin_class) + @plugin_class = plugin_class + end -class CommandRepositoryTest < Test::Unit::TestCase - def setup - @repository = Droonga::CommandRepository.new - end - - class FindTest < self - def setup - super - @command = Droonga::Command.new(:select, - :pattern => ["type", :equal, "select"]) - @repository.register(@command) - end + def pattern + configuration[:pattern] + end - def test_match - assert_equal(@command, @repository.find({ "type" => "select" })) - end + def pattern=(pattern) + configuration[:pattern] = pattern + end - def test_not_match - assert_nil(@repository.find({ "type" => "search" })) + private + def configuration + @plugin_class.options[:message] ||= {} + end + end end end end Deleted: lib/droonga/plugin_registerable.rb (+0 -75) 100644 =================================================================== --- lib/droonga/plugin_registerable.rb 2014-02-17 15:39:00 +0900 (79b64d1) +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2013 Droonga Project -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 2.1 as published by the Free Software Foundation. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -require "droonga/command" -require "droonga/command_repository" -require "droonga/legacy_plugin_repository" - -module Droonga - module PluginRegisterable - class << self - def extended(plugin_class) - super - plugin_class.class_variable_set(:@@repository, - LegacyPluginRepository.new) - end - end - - def repository - class_variable_get(:@@repository) - end - - def inherited(sub_class) - super - sub_class.instance_variable_set(:@command_repository, - CommandRepository.new) - end - - def command(method_name_or_map, options={}) - if method_name_or_map.is_a?(Hash) - type, method_name = method_name_or_map.to_a.first - options[:pattern] ||= ["type", :equal, type.to_s] - else - method_name = method_name_or_map - options[:pattern] ||= ["type", :equal, method_name.to_s] - end - command = Command.new(method_name, options) - @command_repository.register(command) - end - - def commands - @command_repository.commands - end - - def find_command(message) - @command_repository.find(message) - end - - def method_name(message) - message = {"type" => message.to_s} unless message.is_a?(Hash) - command = find_command(message) - return nil if command.nil? - command.method_name - end - - def processable?(message) - message = {"type" => message.to_s} unless message.is_a?(Hash) - command = find_command(message) - not command.nil? - end - end -end Added: lib/droonga/plugins/basic.rb (+53 -0) 100644 =================================================================== --- /dev/null +++ lib/droonga/plugins/basic.rb 2014-02-17 17:28:19 +0900 (52ccdff) @@ -0,0 +1,53 @@ +# Copyright (C) 2014 Droonga Project +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +require "droonga/plugin" +require "droonga/reducer" + +module Droonga + module Plugins + module Basic + Plugin.registry.register("basic", self) + + class GatherCollector < Droonga::Collector + message.pattern = ["task.step.type", :equal, "gather"] + + def collect(message) + output = message.input || message.name + if output.is_a?(Hash) + output_name = output["output"] + else + output_name = output + end + message.values[output_name] = message.value + end + end + + class ReduceCollector < Droonga::Collector + message.pattern = ["task.step.type", :equal, "reduce"] + + def collect(message) + message.input.each do |output_name, deal| + left_value = message.values[output_name] + right_value = message.value + reducer = Reducer.new(deal) + value = reducer.reduce(left_value, right_value) + message.values[output_name] = value + end + end + end + end + end +end Modified: lib/droonga/plugins/search.rb (+90 -0) =================================================================== --- lib/droonga/plugins/search.rb 2014-02-17 15:39:00 +0900 (e49d1ee) +++ lib/droonga/plugins/search.rb 2014-02-17 17:28:19 +0900 (959b777) @@ -45,6 +45,96 @@ module Droonga messenger.emit(values) end end + + class GatherCollector < Droonga::Collector + message.pattern = ["task.step.type", :equal, "search_gather"] + + def collect(message) + output = message.input || message.name + if output.is_a?(Hash) + elements = output["elements"] + if elements and elements.is_a?(Hash) + # because "count" mapper requires all records, + # I have to apply it at first, before "limit" and "offset" are applied. + body = message.body + value = message.value + count_mapper = elements["count"] + if count_mapper + if count_mapper["no_output"] + value.delete("count") + else + value["count"] = value[count_mapper["target"]].size + end + end + + records_mapper = elements["records"] + if records_mapper and value["records"] + if records_mapper["no_output"] + value.delete("records") + else + value["records"] = Reducer.apply_range(value["records"], + records_mapper) + value["records"] = apply_output_attributes_and_format(value["records"], records_mapper) + end + end + end + output_name = output["output"] + else + output_name = output + end + message.values[output_name] = message.value + end + + private + def apply_output_attributes_and_format(items, output) + attributes = output["attributes"] || [] + if output["format"] == "complex" + items.collect! do |item| + complex_item = {} + attributes.each_with_index do |label, index| + complex_item[label] = item[index] + end + complex_item + end + else + items.collect! do |item| + item[0...attributes.size] + end + end + items + end + end + + class ReduceCollector < Droonga::Collector + message.pattern = ["task.step.type", :equal, "search_reduce"] + + def collect(message) + #XXX This is just a workaround. Errors should be handled by the framework itself. + if message.name == "errors" + basic_reduce_collector = Basic::ReduceCollector.new + return basic_reduce_collector.collect(message) + end + + message.input.each do |output_name, elements| + old_value = message.values[output_name] + if old_value + value = reduce_elements(elements, old_value, message.value) + else + value = message.value + end + message.values[output_name] = value + end + end + + def reduce_elements(elements, left_values, right_values) + result = {} + elements.each do |key, deal| + reducer = Reducer.new(deal) + result[key] = reducer.reduce(left_values[key], right_values[key]) + end + result + end + end end end end Renamed: lib/droonga/reducer.rb (+32 -46) 71% =================================================================== --- lib/droonga/plugin/collector/basic.rb 2014-02-17 15:39:00 +0900 (d87df9f) +++ lib/droonga/reducer.rb 2014-02-17 17:28:19 +0900 (454e195) @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2013 Droonga Project +# Copyright (C) 2014 Droonga Project # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -15,78 +13,66 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/collector_plugin" - module Droonga - class BasicCollector < Droonga::CollectorPlugin - repository.register("basic", self) - - UNLIMITED = -1 + class Reducer + class << self + # TODO: This is right location? + def apply_range(items, range) + if items and items.is_a?(Array) + offset = range["offset"] || 0 + unless offset.zero? + items = items[offset..-1] || [] + end - command :gather - def gather(result) - output = body ? body[input_name] : input_name - if output.is_a?(Hash) - output = output["output"] + limit = range["limit"] || 0 + unless limit == UNLIMITED + items = items[0...limit] + end + end + items end - emit(output, result) end - command :reduce - def reduce(request) - body[input_name].each do |output, deal| - left_value = output_values[output] - right_value = request - value = reduce_value(deal, left_value, right_value) - emit(output, value) - end + # XXX: We has ULIMITED defined + # lib/droonga/plugins/search/distributed_search_planner.rb. We + # should unify it. + UNLIMITED = -1 + + def initialize(deal) + @deal = deal # TODO: deal is good name? end - def reduce_value(deal, left_value, right_value) + def reduce(left_value, right_value) if left_value.nil? or right_value.nil? return right_value || left_value end reduced_value = nil - case deal["type"] + case @deal["type"] when "and" reduced_value = (left_value and right_value) when "or" reduced_value = (left_value or right_value) when "sum" reduced_value = sum(left_value, right_value) - reduced_value = apply_output_range(reduced_value, - "limit" => deal["limit"]) + reduced_value = self.class.apply_range(reduced_value, + "limit" => @deal["limit"]) when "average" reduced_value = (left_value.to_f + right_value.to_f) / 2 when "sort" reduced_value = merge(left_value, right_value, - :operators => deal["operators"], - :key_column => deal["key_column"]) - reduced_value = apply_output_range(reduced_value, - "limit" => deal["limit"]) + :operators => @deal["operators"], + :key_column => @deal["key_column"]) + reduced_value = self.class.apply_range(reduced_value, + "limit" => @deal["limit"]) end reduced_value end - def apply_output_range(items, output) - if items and items.is_a?(Array) - offset = output["offset"] || 0 - unless offset.zero? - items = items[offset..-1] || [] - end - - limit = output["limit"] || 0 - unless limit == UNLIMITED - items = items[0...limit] - end - end - items - end - + private def sum(x, y) return x || y if x.nil? or y.nil? Modified: lib/droonga/session.rb (+4 -4) =================================================================== --- lib/droonga/session.rb 2014-02-17 15:39:00 +0900 (c1ffb4b) +++ lib/droonga/session.rb 2014-02-17 17:28:19 +0900 (c7f89aa) @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Droonga Project +# Copyright (C) 2013-2014 Droonga Project # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -15,10 +15,10 @@ module Droonga class Session - def initialize(id, dispatcher, collector, tasks, inputs) + def initialize(id, dispatcher, collector_runner, tasks, inputs) @id = id @dispatcher = dispatcher - @collector = collector + @collector_runner = collector_runner @tasks = tasks @n_dones = 0 @inputs = inputs @@ -56,7 +56,7 @@ module Droonga "name"=>name, "value"=>value } - @collector.process(command, message) + @collector_runner.collect(message) return if task["n_of_inputs"] < n_of_expects #the task is done result = task["values"] Renamed: test/unit/plugins/search/test_collector.rb (+40 -39) 94% =================================================================== --- test/unit/plugin/collector/test_search.rb 2014-02-17 15:39:00 +0900 (1b184d9) +++ test/unit/plugins/search/test_collector.rb 2014-02-17 17:28:19 +0900 (50586bf) @@ -13,16 +13,11 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/plugin/collector/search" +require "droonga/plugins/search" class SearchCollectorTest < Test::Unit::TestCase def setup setup_database - @plugin = Droonga::SearchCollector.new - @outputs = [] - stub(@plugin).emit do |name, value| - @outputs << [name, value] - end end def teardown @@ -34,6 +29,22 @@ class SearchCollectorTest < Test::Unit::TestCase columns end + def run_collector(collector, message) + collector_message = Droonga::CollectorMessage.new(message) + collector.collect(collector_message) + collector_message.values + end + + def gather(message) + collector = Droonga::Plugins::Search::GatherCollector.new + run_collector(collector, message) + end + + def reduce(message) + collector = Droonga::Plugins::Search::ReduceCollector.new + run_collector(collector, message) + end + class << self def create_record(*columns) columns @@ -307,7 +318,7 @@ class SearchCollectorTest < Test::Unit::TestCase def test_gather(data) request = { "task" => { - "values" => nil, + "values" => {}, "step" => { "body" => nil, "outputs" => nil, @@ -318,10 +329,10 @@ class SearchCollectorTest < Test::Unit::TestCase "name" => data[:mapping], "descendants" => nil, } - @plugin.process("search_gather", request) output_name = data[:mapping] output_name = output_name["output"] if output_name.is_a?(Hash) - assert_equal([output_name, data[:expected]], @outputs.last) + assert_equal({ output_name => data[:expected] }, + gather(request)) end end @@ -385,10 +396,8 @@ class SearchCollectorTest < Test::Unit::TestCase "name" => input_name, "descendants" => nil, } - @plugin.process("search_reduce", request) - assert_equal([ - output_name, - { + assert_equal({ + output_name => { "numeric_value" => 3, "numeric_key_records" => [ create_record(1), @@ -407,8 +416,8 @@ class SearchCollectorTest < Test::Unit::TestCase create_record("f"), ], }, - ], - @outputs.last) + }, + reduce(request)) end def test_sum_with_limit @@ -470,10 +479,8 @@ class SearchCollectorTest < Test::Unit::TestCase "name" => input_name, "descendants" => nil, } - @plugin.process("search_reduce", request) - assert_equal([ - output_name, - { + assert_equal({ + output_name => { "numeric_value" => 3, "numeric_key_records" => [ create_record(1), @@ -488,8 +495,8 @@ class SearchCollectorTest < Test::Unit::TestCase create_record("f"), ], }, - ], - @outputs.last) + }, + reduce(request)) end def test_sort @@ -551,10 +558,8 @@ class SearchCollectorTest < Test::Unit::TestCase "name" => input_name, "descendants" => nil, } - @plugin.process("search_reduce", request) - assert_equal([ - output_name, - { + assert_equal({ + output_name => { "numeric_key_records" => [ create_record(1), create_record(2), @@ -572,8 +577,8 @@ class SearchCollectorTest < Test::Unit::TestCase create_record("f"), ], }, - ], - @outputs.last) + }, + reduce(request)) end def test_sort_with_limit @@ -635,10 +640,8 @@ class SearchCollectorTest < Test::Unit::TestCase "name" => input_name, "descendants" => nil, } - @plugin.process("search_reduce", request) - assert_equal([ - output_name, - { + assert_equal({ + output_name => { "numeric_key_records" => [ create_record(1), create_record(2), @@ -652,8 +655,8 @@ class SearchCollectorTest < Test::Unit::TestCase create_record("f"), ], }, - ], - @outputs.last) + }, + reduce(request)) end end @@ -749,10 +752,8 @@ class SearchCollectorTest < Test::Unit::TestCase "name" => input_name, "descendants" => nil, } - @plugin.process("search_reduce", request) - assert_equal([ - output_name, - { + assert_equal({ + output_name => { "records" => [ [ "group3", @@ -798,8 +799,8 @@ class SearchCollectorTest < Test::Unit::TestCase ], ], }, - ], - @outputs.last) + }, + reduce(request)) end end end Renamed: test/unit/plugins/test_basic.rb (+30 -20) 91% =================================================================== --- test/unit/plugin/collector/test_basic.rb 2014-02-17 15:39:00 +0900 (d454e28) +++ test/unit/plugins/test_basic.rb 2014-02-17 17:28:19 +0900 (d8f934e) @@ -13,16 +13,11 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/plugin/collector/basic" +require "droonga/plugins/basic" class BasicCollectorTest < Test::Unit::TestCase def setup setup_database - @plugin = Droonga::BasicCollector.new - @outputs = [] - stub(@plugin).emit do |name, value| - @outputs << [name, value] - end end def teardown @@ -34,6 +29,22 @@ class BasicCollectorTest < Test::Unit::TestCase columns end + def run_collector(collector, message) + collector_message = Droonga::CollectorMessage.new(message) + collector.collect(collector_message) + collector_message.values + end + + def gather(message) + collector = Droonga::Plugins::Basic::GatherCollector.new + run_collector(collector, message) + end + + def reduce(message) + collector = Droonga::Plugins::Basic::ReduceCollector.new + run_collector(collector, message) + end + class << self def create_record(*columns) columns @@ -43,12 +54,12 @@ class BasicCollectorTest < Test::Unit::TestCase class IOTest < self data( :simple_mapping => { - :expected => ["output_name", "result"], + :expected => { "output_name" => "result" }, :source => "result", :mapping => "output_name", }, :complex_mapping => { - :expected => ["output_name", "result"], + :expected => { "output_name" => "result" }, :source => "result", :mapping => { "output" => "output_name", @@ -58,7 +69,7 @@ class BasicCollectorTest < Test::Unit::TestCase def test_gather(data) request = { "task" => { - "values" => nil, + "values" => {}, "step" => { "body" => nil, "outputs" => nil, @@ -69,8 +80,7 @@ class BasicCollectorTest < Test::Unit::TestCase "name" => data[:mapping], "descendants" => nil, } - @plugin.process("gather", request) - assert_equal(data[:expected], @outputs.last) + assert_equal(data[:expected], gather(request)) end def test_reduce @@ -98,18 +108,16 @@ class BasicCollectorTest < Test::Unit::TestCase "name" => input_name, "descendants" => nil, } - @plugin.process("reduce", request) - assert_equal([ - output_name, - [0, 1, 2, 3, 4, 5], - ], - @outputs.last) + assert_equal({ output_name => [0, 1, 2, 3, 4, 5] }, + reduce(request)) end end - class ReduceValueTest < self + # TODO: Split file + class ReducerTest < self def reduce_value(deal, left_value, right_value) - @plugin.reduce_value(deal, left_value, right_value) + reducer = Droonga::Reducer.new(deal) + reducer.reduce(left_value, right_value) end data( @@ -374,9 +382,11 @@ class BasicCollectorTest < Test::Unit::TestCase end end + # TODO: Split file class MergeTest < self def reduce_value(deal, left_value, right_value) - @plugin.reduce_value(deal, left_value, right_value) + reducer = Droonga::Reducer.new(deal) + reducer.reduce(left_value, right_value) end def test_grouped Deleted: test/unit/test_legacy_plugin.rb (+0 -50) 100644 =================================================================== --- test/unit/test_legacy_plugin.rb 2014-02-17 15:39:00 +0900 (9dfc36b) +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (C) 2013 Droonga Project -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 2.1 as published by the Free Software Foundation. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -require "droonga/legacy_plugin" - -class LegacyPluginTest < Test::Unit::TestCase - class PluggableTest < self - class DummyTypePlugin < Droonga::LegacyPlugin - extend Droonga::PluginRegisterable - end - - class DummyPlugin < DummyTypePlugin - command :dummy - def dummy(request) - :dummy_response - end - end - - class UnknownPlugin < DummyTypePlugin - command :unknown - def unknown(request) - :unknown_response - end - end - - def setup - @dummy_plugin = DummyPlugin.new - end - - def test_processable - assert_true(@dummy_plugin.processable?(:dummy)) - end - - def test_not_processable - assert_false(@dummy_plugin.processable?(:unknown)) - end - end -end Deleted: test/unit/test_legacy_plugin_repository.rb (+0 -89) 100644 =================================================================== --- test/unit/test_legacy_plugin_repository.rb 2014-02-17 15:39:00 +0900 (0c71afe) +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (C) 2013 Droonga Project -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 2.1 as published by the Free Software Foundation. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -require "droonga/legacy_plugin_repository" - -class LegacyPluginRepositoryTest < Test::Unit::TestCase - def setup - @repository = Droonga::LegacyPluginRepository.new - end - - class StubPlugin - attr_reader :arguments - def initialize(*arguments) - @arguments = arguments - end - end - - def test_register - @repository.register("stub", StubPlugin) - assert_equal(StubPlugin, @repository["stub"]) - end - - def test_enumerable - @repository.register("stub1", StubPlugin) - @repository.register("stub2", StubPlugin) - assert_equal([ - ["stub1", StubPlugin], - ["stub2", StubPlugin], - ], - @repository.to_a) - end - - sub_test_case("[]") do - def setup - super - @repository.register("stub", StubPlugin) - end - - def test_nonexistent - assert_nil(@repository["nonexistent"]) - end - - def test_existent - assert_equal(StubPlugin, @repository["stub"]) - end - end - - sub_test_case("clear") do - def setup - super - @repository.register("stub", StubPlugin) - end - - def test_clear - assert_equal([["stub", StubPlugin]], @repository.to_a) - @repository.clear - assert_equal([], @repository.to_a) - end - end - - sub_test_case("instantiate") do - def setup - super - @repository.register("stub", StubPlugin) - end - - def test_no_arguments - plugin =****@repos*****("stub") - assert_equal([], plugin.arguments) - end - - def test_have_arguments - plugin =****@repos*****("stub", "Hello", "World") - assert_equal(["Hello", "World"], plugin.arguments) - end - end -end -------------- next part -------------- HTML����������������������������...Download