Kouhei Sutou
null+****@clear*****
Fri Apr 11 17:11:22 JST 2014
Kouhei Sutou 2014-04-11 17:11:22 +0900 (Fri, 11 Apr 2014) New Revision: d3e30f5b35d78887b9f64a9d388a2bdf2f5b2cde https://github.com/droonga/fluent-plugin-droonga/commit/d3e30f5b35d78887b9f64a9d388a2bdf2f5b2cde Message: Apply schema in catalog in slice instead of dispatcher Because schema isn't needed to spread as message. It should be applied on each node. Added files: lib/droonga/schema_applier.rb test/unit/test_schema_applier.rb Modified files: lib/droonga/catalog/schema.rb lib/droonga/dispatcher.rb lib/droonga/farm.rb lib/droonga/slice.rb test/unit/catalog/test_schema.rb Modified: lib/droonga/catalog/schema.rb (+1 -137) =================================================================== --- lib/droonga/catalog/schema.rb 2014-04-11 16:45:22 +0900 (173f486) +++ lib/droonga/catalog/schema.rb 2014-04-11 17:11:22 +0900 (4ac915e) @@ -26,12 +26,6 @@ module Droonga def weight @data["weight"] end - - def flags - flags = [] - flags << "WITH_WEIGHT" if weight - flags - end end class ColumnIndexOptions @@ -54,14 +48,6 @@ module Droonga def sources @data["sources"] end - - def flags - flags = [] - flags << "WITH_SECTION" if section - flags << "WITH_WEIGHT" if weight - flags << "WITH_POSITION" if position - flags - end end class Column @@ -95,23 +81,6 @@ module Droonga end end - def type_flag - case type - when "Scalar" - "COLUMN_SCALAR" - when "Vector" - "COLUMN_VECTOR" - when "Index" - "COLUMN_INDEX" - else - # TODO raise appropriate error - end - end - - def flags - [type_flag] + vector_options.flags + index_options.flags - end - def value_type @data["valueType"] end @@ -124,21 +93,6 @@ module Droonga end end - def to_column_create_body - body = { - "name" => name, - "table" => table, - "flags" => flags.join("|"), - "type" => value_type_groonga - } - sources = index_options.sources - if sources - body["source"] = sources.join(",") - end - - body - end - private def vector_options_data @data["vectorOptions"] || {} @@ -195,7 +149,7 @@ module Droonga when "Float", "Time", "ShortText", "TokyoGeoPoint", "WGS84GeoPoint" key_type else - # TODO raise appropriate error + key_type end end @@ -207,76 +161,12 @@ module Droonga @data["normalizer"] end - def type_flag - case type - when "Array" - "TABLE_NO_KEY" - when "Hash" - "TABLE_HASH_KEY" - when "PatriciaTrie" - "TABLE_PAT_KEY" - when "DoubleArrayTrie" - "TABLE_DAT_KEY" - else - # TODO raise appropriate error - end - end - - def flags - [type_flag] - end - - def to_table_create_body - body = { - "name" => name, - "key_type" => key_type_groonga, - "flags" => flags.join("|") - } - - if tokenizer - body["default_tokenizer"] = tokenizer - end - - if normalizer - body["normalizer"] = normalizer - end - - body - end - private def columns_data @data["columns"] || [] end end - class ColumnCreateSorter - include TSort - - def initialize(tables) - @tables = tables - end - - def all_columns - @tables.values.collect {|table| table.columns.values}.flatten - end - - def tsort_each_node(&block) - all_columns.each(&block) - end - - def tsort_each_child(column, &block) - dependent_column_names = column.index_options.sources || [] - dependent_column_names -= ["_key"] # _key always exists after the table created - reference_table = @tables[column.value_type_groonga] - # TODO when _key specified, check to ensure reference_table is not Array - dependent_columns = dependent_column_names.collect do |column_name| - reference_table.columns[column_name] - end - dependent_columns.each(&block) - end - end - attr_reader :tables def initialize(dataset_name, data) @dataset_name = dataset_name @@ -287,32 +177,6 @@ module Droonga end end - def to_messages - messages = [] - - tables.each do |name, table| - messages << { - "type" => "table_create", - "dataset" => @dataset_name, - "body" => table.to_table_create_body - } - end - - sorter = ColumnCreateSorter.new(tables) - columns = sorter.tsort - # TODO handle TSort::Cyclic - - columns.each do |column| - messages << { - "type" => "column_create", - "dataset" => @dataset_name, - "body" => column.to_column_create_body - } - end - - messages - end - def ==(other) self.class == other.class and tables == other.tables Modified: lib/droonga/dispatcher.rb (+0 -12) =================================================================== --- lib/droonga/dispatcher.rb 2014-04-11 16:45:22 +0900 (2d34a2d) +++ lib/droonga/dispatcher.rb 2014-04-11 17:11:22 +0900 (37a4d01) @@ -62,8 +62,6 @@ module Droonga def start @farm.start - - ensure_schema end def shutdown @@ -268,16 +266,6 @@ module Droonga end end - def ensure_schema - @catalog.datasets.each do |name, dataset| - schema = dataset.schema - messages = schema.to_messages - messages.each do |message| - process_message(message) - end - end - end - def log_tag "[#{Process.ppid}][#{Process.pid}] dispatcher" end Modified: lib/droonga/farm.rb (+3 -1) =================================================================== --- lib/droonga/farm.rb 2014-04-11 16:45:22 +0900 (0c7640a) +++ lib/droonga/farm.rb 2014-04-11 17:11:22 +0900 (b65e0cc) @@ -27,7 +27,9 @@ module Droonga @slices = {} slices =****@catal*****(name) slices.each do |slice_name, slice_options| - slice = Droonga::Slice.new(@loop, + dataset =****@catal*****[slice_options[:dataset]] + slice = Droonga::Slice.new(dataset, + @loop, @options.merge(slice_options)) @slices[slice_name] = slice end Added: lib/droonga/schema_applier.rb (+168 -0) 100644 =================================================================== --- /dev/null +++ lib/droonga/schema_applier.rb 2014-04-11 17:11:22 +0900 (ef60f9b) @@ -0,0 +1,168 @@ +# Copyright (C) 2014 Droonga Project +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +require "droonga/loggable" + +module Droonga + class SchemaApplier + include Loggable + + def initialize(context, schema) + @context = context + @schema = schema + end + + def apply + # TODO: Support migration + Groonga::Schema.define(:context => @context) do |schema| + create_tables(schema) + create_reference_columns(schema) + create_index_columns(schema) + end + end + + private + def each_table + reference_tables = [] + @schema.tables.each_value do |table| + if reference_table?(table) + reference_tables << table + else + yield(table) + end + end + reference_tables.each do |table| + yield(table) + end + end + + def reference_table?(table) + table.type != "Array" and !type_names.include?(table.key_type_groonga) + end + + def each_column(table) + table.columns.each_value do |column| + yield(column) + end + end + + def normal_column?(column) + column.type != "Index" and type_names.include?(column.value_type_groonga) + end + + def reference_column?(column) + column.type != "Index" and !type_names.include?(column.value_type_groonga) + end + + def index_column?(column) + column.type == "Index" + end + + def types + @types ||= collect_available_types + end + + def type_names + @type_names ||= types.collect(&:name) + end + + def collect_available_types + each_options = { + :ignore_missing_object => true + } + @context.database.each(each_options).find_all do |object| + object.is_a?(Groonga::Type) + end + end + + def create_tables(schema) + each_table do |table| + create_table(schema, table) + end + end + + def create_table(schema, table) + options = { + :type => table.type_symbol, + :key_type => table.key_type_groonga, + :default_tokenizer => table.tokenizer, + :normalizer => table.normalizer, + } + p [table.name, options] + schema.create_table(table.name, options) do |table_definition| + each_column(table) do |column| + next unless normal_column?(column) + create_data_column(table_definition, column) + end + end + end + + def create_data_column(table_definition, column) + options = { + :type => column.type_symbol, + } + if options[:type] == :vector + options[:with_weight] = true if column.vector_options.weight + end + table_definition.column(column.name, column.value_type_groonga, options) + end + + def create_reference_columns(schema) + each_table do |table| + reference_columns = [] + each_column(table) do |column| + reference_columns << column if reference_column?(column) + end + next if reference_columns.empty? + + schema.change_table(table.name) do |table_definition| + reference_columns.each do |column| + create_data_column(table_definition, column) + end + end + end + end + + def create_index_columns(schema) + each_table do |table| + index_columns = [] + each_column(table) do |column| + index_columns << column if index_column?(column) + end + next if index_columns.empty? + + schema.change_table(table.name) do |table_definition| + index_columns.each do |column| + create_index_column(table_definition, column) + end + end + end + end + + def create_index_column(table_definition, column) + sources = column.index_options.sources || [] + options = { + :with_section => column.index_options.section, + :with_weight => column.index_options.weight, + :with_position => column.index_options.position, + } + table_definition.index(column.value_type_groonga, *sources, options) + end + + def log_tag + "schema_applier" + end + end +end Modified: lib/droonga/slice.rb (+19 -6) =================================================================== --- lib/droonga/slice.rb 2014-04-11 16:45:22 +0900 (5e6ac27) +++ lib/droonga/slice.rb 2014-04-11 17:11:22 +0900 (76f6f4f) @@ -21,15 +21,17 @@ require "droonga/worker" require "droonga/event_loop" require "droonga/job_pusher" require "droonga/processor" +require "droonga/schema_applier" module Droonga class Slice include Loggable - def initialize(loop, options={}) + def initialize(dataset, loop, options={}) + @dataset = dataset + @loop = loop @options = options @n_workers = @options[:n_workers] || 0 - @loop = loop @job_pusher = JobPusher.new(@loop, @options[:database]) @processor = Processor.new(@loop, @job_pusher, @options) @supervisor = nil @@ -59,12 +61,18 @@ module Droonga private def ensure_database enforce_umask - database_path = @options[:database] - return if File.exist?(database_path) - FileUtils.mkdir_p(File.dirname(database_path)) context = Groonga::Context.new begin - context.create_database(database_path) do + database_path = @options[:database] + if File.exist?(database_path) + context.open_database(database_path) do + apply_schema(context) + end + else + FileUtils.mkdir_p(File.dirname(database_path)) + context.create_database(database_path) do + apply_schema(context) + end end ensure context.close @@ -75,6 +83,11 @@ module Droonga File.umask(022) end + def apply_schema(context) + applier = SchemaApplier.new(context, @dataset.schema) + applier.apply + end + def start_supervisor @supervisor = ServerEngine::Supervisor.new(Server, Worker) do force_options = { Modified: test/unit/catalog/test_schema.rb (+3 -163) =================================================================== --- test/unit/catalog/test_schema.rb 2014-04-11 16:45:22 +0900 (f1265f5) +++ test/unit/catalog/test_schema.rb 2014-04-11 17:11:22 +0900 (df12796) @@ -16,78 +16,11 @@ require "droonga/catalog/schema" class CatalogSchemaTest < Test::Unit::TestCase - private - def create_schema(dataset_name, data) - Droonga::Catalog::Schema.new(dataset_name, data) - end - class SchemaTest < self - def test_schema_not_specified - assert_equal([], - create_schema("dataset_name", nil).to_messages) - end - - def test_no_table - assert_equal([], - create_schema("dataset_name", {}).to_messages) - end - - def test_key_index - assert_equal([ - { - "type" => "table_create", - "dataset" => "dataset_name", - "body" => { - "name" => "Term", - "key_type" => "ShortText", - "flags" => "TABLE_PAT_KEY", - "normalizer" => "NormalizerAuto", - } - }, - { - "type" => "table_create", - "dataset" => "dataset_name", - "body" => { - "name" => "Store", - "key_type" => "ShortText", - "flags" => "TABLE_HASH_KEY", - } - }, - { - "type" => "column_create", - "dataset" => "dataset_name", - "body" => { - "name" => "stores__key", - "table" => "Term", - "type" => "Store", - "flags" => "COLUMN_INDEX", - "source" => "_key" - } - } - ], - create_schema( - "dataset_name", - "Term" => { - "type" => "PatriciaTrie", - "keyType" => "ShortText", - "normalizer" => "NormalizerAuto", - "columns" => { - "stores__key" => { - "type" => "Index", - "valueType" => "Store", - "indexOptions" => { - "sources" => [ - "_key" - ] - } - } - } - }, - "Store" => { - "keyType" => "ShortText" - } - ).to_messages) + def create_schema(dataset_name, data) + Droonga::Catalog::Schema.new(dataset_name, data) end + end class TableTest < self def create_table(name, data) @@ -139,14 +72,6 @@ class CatalogSchemaTest < Test::Unit::TestCase end end - def test_flags - assert_equal(["TABLE_HASH_KEY"], - create_table("table_name", - { - "type" => "Hash" - }).flags) - end - def test_key_type assert_equal("ShortText", create_table("table_name", @@ -178,24 +103,6 @@ class CatalogSchemaTest < Test::Unit::TestCase "normalizer" => "NormalizerAuto" }).normalizer) end - - def test_to_table_create_body - assert_equal({ - "name" => "table_name", - "key_type" => "ShortText", - "flags" => "TABLE_PAT_KEY", - "normalizer" => "NormalizerAuto", - "default_tokenizer" => "TokenBigram" - }, - create_table("table_name", - { - "type" => "PatriciaTrie", - "keyType" => "ShortText", - "normalizer" => "NormalizerAuto", - "tokenizer" => "TokenBigram" - }).to_table_create_body) - - end end class ColumnTest < self @@ -274,56 +181,6 @@ class CatalogSchemaTest < Test::Unit::TestCase assert_equal("Int64", value_type_groonga("Integer")) end end - - def test_to_column_create_body - assert_equal({ - "name" => "column_name", - "flags" => "COLUMN_SCALAR", - "table" => "table_name", - "type" => "ShortText" - }, - create_column("column_name", - { - "type" => "Scalar", - "valueType" => "ShortText" - }).to_column_create_body) - end - - class FlagsTest < self - def flags(data) - create_column("column_name", data).flags - end - - def test_type - data = { - "type" => "Scalar" - } - assert_equal(["COLUMN_SCALAR"], - flags(data)) - end - - def test_weight_options - data = { - "type" => "Vector", - "vectorOptions" => { - "weight" => true - } - } - assert_equal(["COLUMN_VECTOR", "WITH_WEIGHT"], - flags(data)) - end - - def test_index_options - data = { - "type" => "Index", - "indexOptions" => { - "section" => true - } - } - assert_equal(["COLUMN_INDEX", "WITH_SECTION"], - flags(data)) - end - end end class ColumnVectorOptionsTest < self @@ -338,15 +195,6 @@ class CatalogSchemaTest < Test::Unit::TestCase options = create_options(data) assert_equal(true, options.weight) end - - def test_flags - data = { - "weight" => true - } - options = create_options(data) - assert_equal(["WITH_WEIGHT"], - options.flags) - end end class ColumnIndexOptionsTest < self @@ -374,13 +222,5 @@ class CatalogSchemaTest < Test::Unit::TestCase "position" => true }).position) end - - def test_flags - assert_equal(["WITH_SECTION"], - create_options({ - "section" => true - }).flags) - end end - end end Added: test/unit/test_schema_applier.rb (+59 -0) 100644 =================================================================== --- /dev/null +++ test/unit/test_schema_applier.rb 2014-04-11 17:11:22 +0900 (13f91c0) @@ -0,0 +1,59 @@ +# Copyright (C) 2014 Droonga Project +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +require "droonga/schema_applier" + +class SchemaCreatorTest < Test::Unit::TestCase + include Sandbox + + def setup + setup_database + @context = Groonga::Context.default + end + + def teardown + teardown_database + end + + def apply(schema_data) + schema = Droonga::Catalog::Schema.new("dataset", schema_data) + applier = Droonga::SchemaApplier.new(@context, schema) + applier.apply + end + + def dump + dumper = Groonga::SchemaDumper.new(:context => @context, :syntax => :command) + dumper.dump + end + + def test_reference_table + schema_data = { + "Users" => { + "type" => "Hash", + "keyType" => "Names", + }, + "Names" => { + "type" => "Hash", + "keyType" => "ShortText", + }, + } + apply(schema_data) + assert_equal(<<-DUMP, dump) +table_create Names TABLE_HASH_KEY --key_type ShortText + +table_create Users TABLE_HASH_KEY --key_type Names + DUMP + end +end -------------- next part -------------- HTML����������������������������...Download