Kouhei Sutou
null+****@clear*****
Fri Mar 21 16:49:34 JST 2014
Kouhei Sutou 2014-03-21 16:49:34 +0900 (Fri, 21 Mar 2014) New Revision: ab3e1385a2448348fb905bd7d36dbd9b2a7ec0dc https://github.com/droonga/fluent-plugin-droonga/commit/ab3e1385a2448348fb905bd7d36dbd9b2a7ec0dc Message: Move manipulating slice collection code to CollectionVolume from Catalog Modified files: lib/droonga/catalog/collection_volume.rb lib/droonga/catalog/dataset.rb lib/droonga/catalog/slice.rb lib/droonga/catalog/version2.rb lib/droonga/catalog/volume.rb test/unit/catalog/test_collection_volume.rb test/unit/catalog/test_slice.rb test/unit/catalog/test_version2.rb Modified: lib/droonga/catalog/collection_volume.rb (+57 -5) =================================================================== --- lib/droonga/catalog/collection_volume.rb 2014-03-21 16:48:34 +0900 (8b9964b) +++ lib/droonga/catalog/collection_volume.rb 2014-03-21 16:49:34 +0900 (c5d9def) @@ -13,13 +13,18 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +require "digest/sha1" +require "zlib" + require "droonga/catalog/slice" module Droonga module Catalog class CollectionVolume - def initialize(data) + def initialize(dataset, data) + @dataset = dataset @data = data + compute_continuum if ratio_scaled_slicer? end def dimension @@ -32,13 +37,60 @@ module Droonga def slices @slices ||= @data["slices"].collect do |raw_slice| - Slice.new(raw_slice) + Slice.new(@dataset, raw_slice) + end + end + + def select_slices(range=0..-1) + slices.sort_by(&:label)[range] + end + + def choose_slice(record) + return slices.first unless ratio_scaled_slicer? + + key = record[dimension] + hash = Zlib.crc32(key) + min = 0 + max =****@conti***** - 1 + while (min < max) + index = (min + max) / 2 + value, key = @continuum[index] + return key if value == hash + if value > hash + max = index + else + min = index + 1 + end end + @continuum[max][1] end - # Just for backward compatibility - def [](key) - @data[key] + def ratio_scaled_slicer? + slicer == "hash" + end + + private + def compute_continuum + total_weight = compute_total_weight + continuum = [] + n_slices = slices.size + slices.each do |slice| + weight = slice.weight + points = n_slices * 160 * weight / total_weight + points.times do |point| + hash = Digest::SHA1.hexdigest("#{@dataset.name}:#{point}") + continuum << [hash[0..7].to_i(16), slice] + end + end + @continuum = continuum.sort do |a, b| + a[0] - b[0] + end + end + + def compute_total_weight + slices.reduce(0) do |result, slice| + result + slice.weight + end end end end Modified: lib/droonga/catalog/dataset.rb (+1 -1) =================================================================== --- lib/droonga/catalog/dataset.rb 2014-03-21 16:48:34 +0900 (4bceb29) +++ lib/droonga/catalog/dataset.rb 2014-03-21 16:49:34 +0900 (b010492) @@ -61,7 +61,7 @@ module Droonga private def create_volumes(raw_volumes) raw_volumes.collect do |raw_volume| - Volume.create(raw_volume) + Volume.create(self, raw_volume) end end end Modified: lib/droonga/catalog/slice.rb (+3 -2) =================================================================== --- lib/droonga/catalog/slice.rb 2014-03-21 16:48:34 +0900 (1fda8ca) +++ lib/droonga/catalog/slice.rb 2014-03-21 16:49:34 +0900 (537a82c) @@ -18,7 +18,8 @@ require "droonga/catalog/volume" module Droonga module Catalog class Slice - def initialize(data) + def initialize(dataset, data) + @dataset = dataset @data = data end @@ -35,7 +36,7 @@ module Droonga end def volume - @volume ||= Volume.create(@data["volume"]) + @volume ||= Volume.create(@dataset, @data["volume"]) end end end Modified: lib/droonga/catalog/version2.rb (+3 -61) =================================================================== --- lib/droonga/catalog/version2.rb 2014-03-21 16:48:34 +0900 (a181fd2) +++ lib/droonga/catalog/version2.rb 2014-03-21 16:49:34 +0900 (df9f8a6) @@ -64,7 +64,7 @@ module Droonga when "broadcast" volumes = dataset.replicas.select(args["replica"].to_sym) volumes.each do |volume| - slices = select_slices(volume) + slices = volume.select_slices slices.each do |slice| routes << slice.volume.address end @@ -72,10 +72,8 @@ module Droonga when "scatter" volumes = dataset.replicas.select(args["replica"].to_sym) volumes.each do |volume| - dimension = volume.dimension - key = args["key"] || args["record"][dimension] - slice = select_slice(volume, key) - routes << slice["volume"]["address"] + slice = volume.choose_slice(args["record"]) + routes << slice.volume.address end end routes @@ -90,65 +88,9 @@ module Droonga def prepare_data @datasets = {} @data["datasets"].each do |name, dataset| - replicas = dataset["replicas"] - replicas.each do |replica| - total_weight = compute_total_weight(replica) - continuum = [] - slices = replica["slices"] - n_slices = slices.size - slices.each do |slice| - weight = slice["weight"] || default_weight - points = n_slices * 160 * weight / total_weight - points.times do |point| - hash = Digest::SHA1.hexdigest("#{name}:#{point}") - continuum << [hash[0..7].to_i(16), slice] - end - end - replica["continuum"] = continuum.sort do |a, b| - a[0] - b[0] - end - end @datasets[name] = Dataset.new(name, dataset) end end - - def default_weight - 1 - end - - def compute_total_weight(replica) - slices = replica["slices"] - slices.reduce(0) do |result, slice| - result + (slice["weight"] || default_weight) - end - end - - def select_slices(volume, range=0..-1) - sorted_slices = volume.slices.sort_by do |slice| - slice.label - end - sorted_slices[range] - end - - def select_slice(volume, key) - continuum = volume["continuum"] - return volume.slices.first unless continuum - - hash = Zlib.crc32(key) - min = 0 - max = continuum.size - 1 - while (min < max) - index = (min + max) / 2 - value, key = continuum[index] - return key if value == hash - if value > hash - max = index - else - min = index + 1 - end - end - continuum[max][1] - end end end end Modified: lib/droonga/catalog/volume.rb (+2 -2) =================================================================== --- lib/droonga/catalog/volume.rb 2014-03-21 16:48:34 +0900 (25f7e5e) +++ lib/droonga/catalog/volume.rb 2014-03-21 16:49:34 +0900 (e56a096) @@ -20,11 +20,11 @@ module Droonga module Catalog module Volume class << self - def create(raw_volume) + def create(dataset, raw_volume) if raw_volume.key?("address") SingleVolume.new(raw_volume) else - CollectionVolume.new(raw_volume) + CollectionVolume.new(dataset, raw_volume) end end end Modified: test/unit/catalog/test_collection_volume.rb (+39 -2) =================================================================== --- test/unit/catalog/test_collection_volume.rb 2014-03-21 16:48:34 +0900 (9c006d5) +++ test/unit/catalog/test_collection_volume.rb 2014-03-21 16:49:34 +0900 (d0f283d) @@ -13,16 +13,22 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/catalog/collection_volume" +require "droonga/catalog/dataset" class CatalogSingleVolumeTest < Test::Unit::TestCase def create_collection_volume(data) - Droonga::Catalog::CollectionVolume.new(data) + minimum_dataset_data = { + "replicas" => { + }, + } + dataset = Droonga::Catalog::Dataset.new("DatasetName", minimum_dataset_data) + Droonga::Catalog::CollectionVolume.new(dataset, data) end class DimensionTest < self def test_default data = { + "slices" => [], } volume = create_collection_volume(data) assert_equal("_key", volume.dimension) @@ -31,6 +37,7 @@ class CatalogSingleVolumeTest < Test::Unit::TestCase def test_specified data = { "dimension" => "group", + "slices" => [], } volume = create_collection_volume(data) assert_equal("group", volume.dimension) @@ -40,6 +47,7 @@ class CatalogSingleVolumeTest < Test::Unit::TestCase class SlicerTest < self def test_default data = { + "slices" => [], } volume = create_collection_volume(data) assert_equal("hash", volume.slicer) @@ -63,4 +71,33 @@ class CatalogSingleVolumeTest < Test::Unit::TestCase assert_equal([], volume.slices) end end + + class RatioOrderSlicerTest < self + class TotalWeightTest < self + def test_three_slices + data = { + "slicer" => "hash", + "slices" => [ + { + "weight" => 10, + }, + { + "weight" => 20, + }, + { + "weight" => 30, + }, + ], + } + assert_equal(10 + 20 + 30, + total_weight(data)) + end + + private + def total_weight(data) + volume = create_collection_volume(data) + volume.send(:compute_total_weight) + end + end + end end Modified: test/unit/catalog/test_slice.rb (+7 -2) =================================================================== --- test/unit/catalog/test_slice.rb 2014-03-21 16:48:34 +0900 (23aeaaf) +++ test/unit/catalog/test_slice.rb 2014-03-21 16:49:34 +0900 (ae4d842) @@ -13,12 +13,17 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -require "droonga/catalog/slice" +require "droonga/catalog/dataset" class CatalogSliceTest < Test::Unit::TestCase private def create_slice(data) - Droonga::Catalog::Slice.new(data) + minimum_dataset_data = { + "replicas" => { + }, + } + dataset = Droonga::Catalog::Dataset.new("DatasetName", minimum_dataset_data) + Droonga::Catalog::Slice.new(dataset, data) end class WeightTest < self Modified: test/unit/catalog/test_version2.rb (+0 -31) =================================================================== --- test/unit/catalog/test_version2.rb 2014-03-21 16:48:34 +0900 (6de915f) +++ test/unit/catalog/test_version2.rb 2014-03-21 16:49:34 +0900 (9ecf370) @@ -121,35 +121,4 @@ class CatalogVersion2Test < Test::Unit::TestCase end end end - - class DataSetTest < self - class ReplicaTest < self - class TotalWeightTest < self - def test_three_slices - replica = { - "slices" => [ - { - "weight" => 10, - }, - { - "weight" => 20, - }, - { - "weight" => 30, - }, - ], - } - assert_equal(10 + 20 + 30, - total_weight(replica)) - end - - private - def total_weight(replica) - catalog = create_catalog(minimum_data, - "base-path") - catalog.send(:compute_total_weight, replica) - end - end - end - end end -------------- next part -------------- HTML����������������������������...Download