Kouhei Sutou
null+****@clear*****
Thu Nov 6 00:45:45 JST 2014
Kouhei Sutou 2014-11-06 00:45:45 +0900 (Thu, 06 Nov 2014) New Revision: 906d9dc33a0b9a9905220ce9b205d9b05e8754f6 https://github.com/droonga/droonga-engine/commit/906d9dc33a0b9a9905220ce9b205d9b05e8754f6 Message: Optimize for single slice case We don't need to reduce for single slice case. So we don't need to return all records. TODO: * Clean unit test. We should use real dataset object. Find TODO comments. Modified files: lib/droonga/catalog/dataset.rb lib/droonga/distributed_command_planner.rb lib/droonga/planner.rb lib/droonga/plugins/search.rb lib/droonga/plugins/search/distributed_search_planner.rb lib/droonga/single_step.rb test/unit/helper/distributed_search_planner_helper.rb test/unit/plugins/search/planner/test_basic.rb test/unit/plugins/search/test_planner.rb Modified: lib/droonga/catalog/dataset.rb (+8 -0) =================================================================== --- lib/droonga/catalog/dataset.rb 2014-11-06 00:38:07 +0900 (ca4b8a5) +++ lib/droonga/catalog/dataset.rb 2014-11-06 00:45:45 +0900 (6707277) @@ -85,6 +85,14 @@ module Droonga routes end + def single_slice? + # TODO: Support slice key + replicas.all? do |volume| + volume.is_a?(SingleVolume) or + volume.slices.size == 1 + end + end + private def create_volumes(raw_volumes) raw_volumes.collect do |raw_volume| Modified: lib/droonga/distributed_command_planner.rb (+5 -5) =================================================================== --- lib/droonga/distributed_command_planner.rb 2014-11-06 00:38:07 +0900 (a462dd3) +++ lib/droonga/distributed_command_planner.rb 2014-11-06 00:45:45 +0900 (a1f5102) @@ -17,17 +17,17 @@ module Droonga class DistributedCommandPlanner - attr_accessor :key, :dataset + attr_accessor :key REDUCE_SUM = "sum" DEFAULT_LIMIT = -1 - def initialize(source_message) + def initialize(dataset, source_message) + @dataset = dataset @source_message = source_message @key = nil - @dataset = nil @outputs = [] @reducers = [] @@ -58,7 +58,7 @@ module Droonga def scatter(record, options={}) @processor = { "command" => @source_message["type"], - "dataset" => @dataset || @source_message["dataset"], + "dataset" => @dataset.name, "body" => options[:body] || @source_message["body"], "record" => record, "type" => "scatter", @@ -71,7 +71,7 @@ module Droonga def broadcast(options={}) processor = { "command" => @source_message["type"], - "dataset" => @dataset || @source_message["dataset"], + "dataset" => @dataset.name, "body" => options[:body] || @source_message["body"], "type" => "broadcast", "outputs" => [], Modified: lib/droonga/planner.rb (+4 -3) =================================================================== --- lib/droonga/planner.rb 2014-11-06 00:38:07 +0900 (297c1e6) +++ lib/droonga/planner.rb 2014-11-06 00:45:45 +0900 (532b52b) @@ -22,7 +22,8 @@ module Droonga include Loggable include ErrorMessages - def initialize + def initialize(dataset) + @dataset = dataset end def plan(message) @@ -31,14 +32,14 @@ module Droonga private def scatter(message, record, options={}) - planner = DistributedCommandPlanner.new(message) + planner = DistributedCommandPlanner.new(@dataset, message) planner.scatter(record) planner.reduce(options[:reduce]) planner.plan end def broadcast(message, options={}) - planner = DistributedCommandPlanner.new(message) + planner = DistributedCommandPlanner.new(@dataset, message) planner.broadcast(:write => options[:write]) planner.reduce(options[:reduce]) planner.plan Modified: lib/droonga/plugins/search.rb (+1 -1) =================================================================== --- lib/droonga/plugins/search.rb 2014-11-06 00:38:07 +0900 (b5980ab) +++ lib/droonga/plugins/search.rb 2014-11-06 00:45:45 +0900 (850c82f) @@ -25,7 +25,7 @@ module Droonga class Planner < Droonga::Planner def plan(message) - planner = DistributedSearchPlanner.new(message) + planner = DistributedSearchPlanner.new(@dataset, message) planner.plan end end Modified: lib/droonga/plugins/search/distributed_search_planner.rb (+10 -6) =================================================================== --- lib/droonga/plugins/search/distributed_search_planner.rb 2014-11-06 00:38:07 +0900 (86cc605) +++ lib/droonga/plugins/search/distributed_search_planner.rb 2014-11-06 00:45:45 +0900 (06766fa) @@ -22,7 +22,7 @@ module Droonga module Plugins module Search class DistributedSearchPlanner < DistributedCommandPlanner - def initialize(search_request_message) + def initialize(dataset, search_request_message) super @request = @source_message["body"] @@ -43,7 +43,6 @@ module Droonga transform_query(input_name, query) end - @dataset = @source_message["dataset"] || @request["dataset"] broadcast(:body => @request) super @@ -79,7 +78,7 @@ module Droonga def transform_query(input_name, query) return unless need_reduce?(query) - transformer = QueryTransformer.new(query) + transformer = QueryTransformer.new(@dataset, query) elements = transformer.mappers mapper = {} mapper["elements"] = elements unless elements.empty? @@ -103,7 +102,8 @@ module Droonga class QueryTransformer attr_reader :reducers, :mappers - def initialize(query) + def initialize(dataset, query) + @dataset = dataset @query = query @output = @query["output"] @reducers = {} @@ -210,10 +210,14 @@ module Droonga end def final_offset + return @original_output_offset if****@datas*****_slice? + @original_sort_offset + @original_output_offset end def final_limit + return @original_output_limit if****@datas*****_slice? + if @original_sort_limit == UNLIMITED and @original_output_limit == UNLIMITED UNLIMITED @@ -248,12 +252,12 @@ module Droonga } if unifiable? @query["sortBy"]["limit"] = -1 if @query["sortBy"].is_a?(Hash) - @output["limit"] = -1 + @output["limit"] = -1 unles****@datas*****_slice? mapper = { "target" => "records", } unless @output["elements"].include?("records") - @records_limit = -1 + @records_limit = -1 unles****@datas*****_slice? @output["elements"] << "records" @output["attributes"] ||= ["_key"] @output_records = false Modified: lib/droonga/single_step.rb (+2 -2) =================================================================== --- lib/droonga/single_step.rb 2014-11-06 00:38:07 +0900 (3aa3c48) +++ lib/droonga/single_step.rb 2014-11-06 00:45:45 +0900 (74075b2) @@ -26,12 +26,12 @@ module Droonga def plan(message) if message["type"] == "search" # XXX: workaround - planner = Plugins::Search::Planner.new + planner = Plugins::Search::Planner.new(@dataset) return planner.plan(message) end # XXX: Re-implement me. - planner = Planner.new + planner = Planner.new(@dataset) options = {} options[:write] =****@defin*****? collector_class =****@defin*****_class Modified: test/unit/helper/distributed_search_planner_helper.rb (+9 -1) =================================================================== --- test/unit/helper/distributed_search_planner_helper.rb 2014-11-06 00:38:07 +0900 (2daa82c) +++ test/unit/helper/distributed_search_planner_helper.rb 2014-11-06 00:45:45 +0900 (6c36a21) @@ -17,7 +17,15 @@ require "droonga/plugins/search/distributed_search_planner" module DistributedSearchPlannerHelper def plan(search_request) - planner = Droonga::Plugins::Search::DistributedSearchPlanner.new(search_request) + # TODO: Use real dataset + stub_dataset = Object.new + stub(stub_dataset).name do + Droonga::Catalog::Dataset::DEFAULT_NAME + end + stub(stub_dataset).single_slice? do + false + end + planner = Droonga::Plugins::Search::DistributedSearchPlanner.new(stub_dataset, search_request) planner.plan end Modified: test/unit/plugins/search/planner/test_basic.rb (+2 -2) =================================================================== --- test/unit/plugins/search/planner/test_basic.rb 2014-11-06 00:38:07 +0900 (64153b6) +++ test/unit/plugins/search/planner/test_basic.rb 2014-11-06 00:45:45 +0900 (b325b90) @@ -21,7 +21,7 @@ class DistributedSearchPlannerBasicTest < Test::Unit::TestCase def setup @request = { "type" => "search", - "dataset" => "Droonga", + "dataset" => "Default", "body" => { "queries" => { "query1" => { @@ -63,7 +63,7 @@ class DistributedSearchPlannerBasicTest < Test::Unit::TestCase def setup @request = { "type" => "search", - "dataset" => "Droonga", + "dataset" => "Default", "body" => { "queries" => { "no_output" => { Modified: test/unit/plugins/search/test_planner.rb (+10 -2) =================================================================== --- test/unit/plugins/search/test_planner.rb 2014-11-06 00:38:07 +0900 (e7e3bf7) +++ test/unit/plugins/search/test_planner.rb 2014-11-06 00:45:45 +0900 (0a2b35e) @@ -19,7 +19,15 @@ class SearchPlannerTest < Test::Unit::TestCase def setup setup_database @planner = Droonga::Test::StubPlanner.new - @plugin = Droonga::Plugins::Search::Planner.new + # TODO: Use real dataset + stub_dataset = Object.new + stub(stub_dataset).name do + Droonga::Catalog::Dataset::DEFAULT_NAME + end + stub(stub_dataset).single_slice? do + false + end + @plugin = Droonga::Plugins::Search::Planner.new(stub_dataset) end def teardown @@ -135,7 +143,7 @@ class SearchPlannerTest < Test::Unit::TestCase searcher = { "type" => "broadcast", "command" => "search", - "dataset" => "Droonga", + "dataset" => "Default", "body" => { "queries" => { "query1" => { -------------- next part -------------- HTML����������������������������...Download