YUKI Hiroshi
null+****@clear*****
Thu Dec 4 20:48:31 JST 2014
YUKI Hiroshi 2014-12-04 20:48:31 +0900 (Thu, 04 Dec 2014) New Revision: a5f37edced7aeaf08db4828dba0619c181f83096 https://github.com/droonga/droonga-engine/commit/a5f37edced7aeaf08db4828dba0619c181f83096 Merged e0f9f34: Merge pull request #30 from piroor/replicas-in-a-branch Message: Support replicas in a slice ex: replicas: [ { // replica slices: [ { // slice replicas: [ { volume: ... }, // replica of a slice { volume: ... } ] } ] } ] Copied files: lib/droonga/catalog/replica.rb (from lib/droonga/catalog/slice.rb) Modified files: lib/droonga/catalog/dataset.rb lib/droonga/catalog/slice.rb lib/droonga/catalog/volume.rb test/unit/catalog/test_dataset.rb test/unit/catalog/test_slice.rb Renamed files: lib/droonga/catalog/replicas.rb (from lib/droonga/catalog/volume_collection.rb) lib/droonga/catalog/slices.rb (from lib/droonga/catalog/collection_volume.rb) test/unit/catalog/test_replicas.rb (from test/unit/catalog/test_volume_collection.rb) test/unit/catalog/test_slices.rb (from test/unit/catalog/test_collection_volume.rb) Modified: lib/droonga/catalog/dataset.rb (+31 -26) =================================================================== --- lib/droonga/catalog/dataset.rb 2014-12-02 12:44:16 +0900 (6707277) +++ lib/droonga/catalog/dataset.rb 2014-12-04 20:48:31 +0900 (e92b6a8) @@ -14,8 +14,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA require "droonga/catalog/schema" -require "droonga/catalog/volume" -require "droonga/catalog/volume_collection" +require "droonga/catalog/replicas" module Droonga module Catalog @@ -57,7 +56,7 @@ module Droonga end def replicas - @replicas ||= VolumeCollection.new(create_volumes(@data["replicas"])) + @replicas ||= Replicas.create(self, @data["replicas"]) end def all_nodes @@ -65,40 +64,46 @@ module Droonga end def compute_routes(message, live_nodes) + compute_routes_from_replicas(replicas, message, live_nodes) + end + + def single_slice? + # TODO: Support slice key + replicas.all? do |replica| + replica.is_a?(SingleVolume) or + replica.slices.size == 1 + end + end + + private + def compute_routes_from_replicas(replicas, message, live_nodes) routes = [] case message["type"] when "broadcast" - volumes = replicas.select(message["replica"].to_sym, live_nodes) - volumes.each do |volume| - slices = volume.select_slices + replicas = replicas.select(message["replica"].to_sym, live_nodes) + replicas.each do |replica| + slices = replica.select_slices slices.each do |slice| - routes << slice.volume.address.to_s + if slice.replicas + routes += compute_routes_from_replicas(slice.replicas, message, live_nodes) + else + routes << slice.volume.address.to_s + end end end when "scatter" - volumes = replicas.select(message["replica"].to_sym, live_nodes) - volumes.each do |volume| - slice = volume.choose_slice(message["record"]) - routes << slice.volume.address.to_s + replicas = replicas.select(message["replica"].to_sym, live_nodes) + replicas.each do |replica| + slice = replica.choose_slice(message["record"]) + if slice.replicas + routes += compute_routes_from_replicas(slice.replicas, message, live_nodes) + else + routes << slice.volume.address.to_s + end end end routes end - - def single_slice? - # TODO: Support slice key - replicas.all? do |volume| - volume.is_a?(SingleVolume) or - volume.slices.size == 1 - end - end - - private - def create_volumes(raw_volumes) - raw_volumes.collect do |raw_volume| - Volume.create(self, raw_volume) - end - end end end end Copied: lib/droonga/catalog/replica.rb (+8 -20) 71% =================================================================== --- lib/droonga/catalog/slice.rb 2014-12-02 12:44:16 +0900 (d9877f4) +++ lib/droonga/catalog/replica.rb 2014-12-04 20:48:31 +0900 (a97270a) @@ -13,32 +13,20 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +require "droonga/catalog/slices" + module Droonga module Catalog - class Slice - def initialize(dataset, data) - @dataset = dataset - @data = data - end - - def weight - @data["weight"] || 1 - end - - def label - @data["label"] - end - - def boundary - @data["boundary"] - end - + class Replica < Slices def volume + return nil unles****@data*****?("volume") @volume ||= Volume.create(@dataset, @data["volume"]) end - def all_nodes - @all_nodes ||= volume.all_nodes + private + def collect_all_nodes + return volume.all_nodes if volume + super end end end Renamed: lib/droonga/catalog/replicas.rb (+25 -14) 65% =================================================================== --- lib/droonga/catalog/volume_collection.rb 2014-12-02 12:44:16 +0900 (eacf173) +++ lib/droonga/catalog/replicas.rb 2014-12-04 20:48:31 +0900 (4c3a6e5) @@ -13,17 +13,28 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +require "droonga/catalog/volume" + module Droonga module Catalog - class VolumeCollection + class Replicas + class << self + def create(dataset, raw_replicas) + replicas = raw_replicas.collect do |raw_replica| + Replica.new(dataset, raw_replica) + end + new(replicas) + end + end + include Enumerable - def initialize(volumes) - @volumes = volumes + def initialize(replicas) + @replicas = replicas end def each(&block) - @volumes.each(&block) + @replicas.each(&block) end def ==(other) @@ -40,14 +51,14 @@ module Droonga end def select(how=nil, live_nodes=nil) - volumes = live_volumes(live_nodes) + replicas = live_replicas(live_nodes) case how when :top - [volumes.first] + [replicas.first] when :random - [volumes.sample] + [replicas.sample] when :all - @volumes + @replicas else super end @@ -57,11 +68,11 @@ module Droonga @all_nodes ||= collect_all_nodes end - def live_volumes(live_nodes=nil) - return @volumes unless live_nodes + def live_replicas(live_nodes=nil) + return @replicas unless live_nodes - @volumes.select do |volume| - dead_nodes = volume.all_nodes - live_nodes + @replicas.select do |replica| + dead_nodes = replica.all_nodes - live_nodes dead_nodes.empty? end end @@ -69,8 +80,8 @@ module Droonga private def collect_all_nodes nodes = [] - @volumes.each do |volume| - nodes += volume.all_nodes + @replicas.each do |replica| + nodes += replica.all_nodes end nodes.sort.uniq end Modified: lib/droonga/catalog/slice.rb (+16 -1) =================================================================== --- lib/droonga/catalog/slice.rb 2014-12-02 12:44:16 +0900 (d9877f4) +++ lib/droonga/catalog/slice.rb 2014-12-04 20:48:31 +0900 (59ab1ef) @@ -34,11 +34,26 @@ module Droonga end def volume + return nil unles****@data*****?("volume") @volume ||= Volume.create(@dataset, @data["volume"]) end + def replicas + return nil unles****@data*****?("replicas") + @replicas ||= Replicas.create(@dataset, @data["replicas"]) + end + def all_nodes - @all_nodes ||= volume.all_nodes + @all_nodes ||= collect_all_nodes + end + + private + def collect_all_nodes + if volume + volume.all_nodes + else + replicas.all_nodes + end end end end Renamed: lib/droonga/catalog/slices.rb (+3 -2) 95% =================================================================== --- lib/droonga/catalog/collection_volume.rb 2014-12-02 12:44:16 +0900 (3f3df7d) +++ lib/droonga/catalog/slices.rb 2014-12-04 20:48:31 +0900 (5c6e15f) @@ -20,11 +20,11 @@ require "droonga/catalog/slice" module Droonga module Catalog - class CollectionVolume + class Slices def initialize(dataset, data) @dataset = dataset @data = data - compute_continuum if ratio_scaled_slicer? + compute_continuum if slices and ratio_scaled_slicer? end def dimension @@ -36,6 +36,7 @@ module Droonga end def slices + return nil unles****@data*****?("slices") @slices ||= @data["slices"].collect do |raw_slice| Slice.new(@dataset, raw_slice) end Modified: lib/droonga/catalog/volume.rb (+2 -2) =================================================================== --- lib/droonga/catalog/volume.rb 2014-12-02 12:44:16 +0900 (1a40ca0) +++ lib/droonga/catalog/volume.rb 2014-12-04 20:48:31 +0900 (9262344) @@ -14,7 +14,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA require "droonga/catalog/single_volume" -require "droonga/catalog/collection_volume" +require "droonga/catalog/replica" module Droonga module Catalog @@ -24,7 +24,7 @@ module Droonga if raw_volume.key?("address") SingleVolume.new(raw_volume) else - CollectionVolume.new(dataset, raw_volume) + Replica.new(dataset, raw_volume) end end end Modified: test/unit/catalog/test_dataset.rb (+1 -1) =================================================================== --- test/unit/catalog/test_dataset.rb 2014-12-02 12:44:16 +0900 (d4476b6) +++ test/unit/catalog/test_dataset.rb 2014-12-04 20:48:31 +0900 (47b551b) @@ -97,7 +97,7 @@ class CatalogDatasetTest < Test::Unit::TestCase "replicas" => [], } dataset = create_dataset(data) - assert_equal(Droonga::Catalog::VolumeCollection.new([]), + assert_equal(Droonga::Catalog::Replicas.new([]), dataset.replicas) end end Renamed: test/unit/catalog/test_replicas.rb (+10 -10) 80% =================================================================== --- test/unit/catalog/test_volume_collection.rb 2014-12-02 12:44:16 +0900 (93f9bd4) +++ test/unit/catalog/test_replicas.rb 2014-12-04 20:48:31 +0900 (d25767f) @@ -13,12 +13,12 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -require "droonga/catalog/volume_collection" +require "droonga/catalog/replicas" -class CatalogVolumeCollectionTest < Test::Unit::TestCase +class CatalogReplicasTest < Test::Unit::TestCase private - def create_volume_collection(volumes) - Droonga::Catalog::VolumeCollection.new(volumes) + def create_replicas(replicas) + Droonga::Catalog::Replicas.new(replicas) end class SelectTest < self @@ -28,7 +28,7 @@ class CatalogVolumeCollectionTest < Test::Unit::TestCase "volume2", "volume3", ] - @collection = create_volume_collection(volumes) + @collection = create_replicas(volumes) end def test_top @@ -49,11 +49,11 @@ class CatalogVolumeCollectionTest < Test::Unit::TestCase end class NodesTest < self - def create_volume_collection(volumes) - volumes = volumes.collect do |volume| - create_single_volume(volume) + def create_replicas(raw_replicas) + replicas = raw_replicas.collect do |replica| + create_single_volume(replica) end - super(volumes) + super(replicas) end def create_single_volume(data) @@ -67,7 +67,7 @@ class CatalogVolumeCollectionTest < Test::Unit::TestCase { "address" => "volume2:10047/droonga.002" }, { "address" => "volume2:10047/droonga.003" }, ] - @collection = create_volume_collection(volumes) + @collection = create_replicas(volumes) end def test_all_nodes Modified: test/unit/catalog/test_slice.rb (+15 -0) =================================================================== --- test/unit/catalog/test_slice.rb 2014-12-02 12:44:16 +0900 (d02897b) +++ test/unit/catalog/test_slice.rb 2014-12-04 20:48:31 +0900 (96ed049) @@ -99,5 +99,20 @@ class CatalogSliceTest < Test::Unit::TestCase assert_equal(["127.0.0.1:10047/volume"], slice.all_nodes) end + + def test_deeply_nested + data = { + "replicas" => [ + { + "volume" => { + "address" => "127.0.0.1:10047/volume.000", + }, + }, + ], + } + slice = create_slice(data) + assert_equal(["127.0.0.1:10047/volume"], + slice.all_nodes) + end end end Renamed: test/unit/catalog/test_slices.rb (+31 -9) 73% =================================================================== --- test/unit/catalog/test_collection_volume.rb 2014-12-02 12:44:16 +0900 (346dd86) +++ test/unit/catalog/test_slices.rb 2014-12-04 20:48:31 +0900 (db5c7ae) @@ -16,13 +16,13 @@ require "droonga/catalog/dataset" class CatalogSingleVolumeTest < Test::Unit::TestCase - def create_collection_volume(data) + def create_slices(data) minimum_dataset_data = { "replicas" => { }, } dataset = Droonga::Catalog::Dataset.new("DatasetName", minimum_dataset_data) - Droonga::Catalog::CollectionVolume.new(dataset, data) + Droonga::Catalog::Slices.new(dataset, data) end class DimensionTest < self @@ -30,7 +30,7 @@ class CatalogSingleVolumeTest < Test::Unit::TestCase data = { "slices" => [], } - volume = create_collection_volume(data) + volume = create_slices(data) assert_equal("_key", volume.dimension) end @@ -39,7 +39,7 @@ class CatalogSingleVolumeTest < Test::Unit::TestCase "dimension" => "group", "slices" => [], } - volume = create_collection_volume(data) + volume = create_slices(data) assert_equal("group", volume.dimension) end end @@ -49,7 +49,7 @@ class CatalogSingleVolumeTest < Test::Unit::TestCase data = { "slices" => [], } - volume = create_collection_volume(data) + volume = create_slices(data) assert_equal("hash", volume.slicer) end @@ -57,7 +57,7 @@ class CatalogSingleVolumeTest < Test::Unit::TestCase data = { "slicer" => "ordinal", } - volume = create_collection_volume(data) + volume = create_slices(data) assert_equal("ordinal", volume.slicer) end end @@ -67,7 +67,7 @@ class CatalogSingleVolumeTest < Test::Unit::TestCase data = { "slices" => [], } - volume = create_collection_volume(data) + volume = create_slices(data) assert_equal([], volume.slices) end end @@ -95,7 +95,7 @@ class CatalogSingleVolumeTest < Test::Unit::TestCase private def total_weight(data) - volume = create_collection_volume(data) + volume = create_slices(data) volume.send(:compute_total_weight) end end @@ -111,7 +111,29 @@ class CatalogSingleVolumeTest < Test::Unit::TestCase { "volume" => { "address" => "127.0.0.1:23004/droonga.101" } }, ], } - volume = create_collection_volume(data) + volume = create_slices(data) + assert_equal(["127.0.0.1:23003/droonga", "127.0.0.1:23004/droonga"], + volume.all_nodes) + end + + def test_deeply_nested + data = { + "slices" => [ + { + "replicas" => [ + { "volume" => { "address" => "127.0.0.1:23003/droonga.000" } }, + { "volume" => { "address" => "127.0.0.1:23003/droonga.001" } }, + ], + }, + { + "replicas" => [ + { "volume" => { "address" => "127.0.0.1:23004/droonga.100" } }, + { "volume" => { "address" => "127.0.0.1:23004/droonga.101" } }, + ], + }, + ], + } + volume = create_slices(data) assert_equal(["127.0.0.1:23003/droonga", "127.0.0.1:23004/droonga"], volume.all_nodes) end -------------- next part -------------- HTML����������������������������...Download