Kouhei Sutou
null+****@clear*****
Thu Mar 16 14:48:33 JST 2017
Kouhei Sutou 2017-03-16 14:48:33 +0900 (Thu, 16 Mar 2017) New Revision: 43cea82e76edb990630fdbbddaa43e378bd83a34 https://github.com/groonga/groonga/commit/43cea82e76edb990630fdbbddaa43e378bd83a34 Message: logical_select: support match_columns and query Added files: test/command/suite/sharding/logical_select/query/column.expected test/command/suite/sharding/logical_select/query/column.test test/command/suite/sharding/logical_select/query/with_filter.expected test/command/suite/sharding/logical_select/query/with_filter.test test/command/suite/sharding/logical_select/query/with_match_columns.expected test/command/suite/sharding/logical_select/query/with_match_columns.test Modified files: plugins/sharding/logical_count.rb plugins/sharding/logical_range_filter.rb plugins/sharding/logical_select.rb plugins/sharding/logical_table_remove.rb plugins/sharding/range_expression_builder.rb Modified: plugins/sharding/logical_count.rb (+2 -2) =================================================================== --- plugins/sharding/logical_count.rb 2017-03-16 14:46:40 +0900 (1d94750) +++ plugins/sharding/logical_count.rb 2017-03-16 14:48:33 +0900 (8bdd77e) @@ -66,8 +66,8 @@ module Groonga table_name = shard.table_name expression_builder = RangeExpressionBuilder.new(shard_key, - target_range, - filter) + target_range) + expression_builder.filter = filter if cover_type == :all log_use_range_index(false, table_name, __LINE__, __method__) if filter.nil? Modified: plugins/sharding/logical_range_filter.rb (+2 -2) =================================================================== --- plugins/sharding/logical_range_filter.rb 2017-03-16 14:46:40 +0900 (6ccbe86) +++ plugins/sharding/logical_range_filter.rb 2017-03-16 14:48:33 +0900 (1c8f864) @@ -211,8 +211,8 @@ module Groonga end expression_builder = RangeExpressionBuilder.new(shard_key, - @target_range, - @filter) + @target_range) + expression_builder.filter = @filter index_info = shard_key.find_index(Operator::LESS) if index_info Modified: plugins/sharding/logical_select.rb (+66 -31) =================================================================== --- plugins/sharding/logical_select.rb 2017-03-16 14:46:40 +0900 (d7b9bf6) +++ plugins/sharding/logical_select.rb 2017-03-16 14:48:33 +0900 (941e3b2) @@ -25,6 +25,8 @@ module Groonga "drilldown_calc_target", "sort_keys", "drilldown_sort_keys", + "match_columns", + "query", ]) def run_body(input) @@ -76,6 +78,8 @@ module Groonga key << "#{input[:limit]}\0" key << "#{input[:drilldown]}\0" key << "#{drilldown_sort_keys}\0" + key << "#{input[:match_columns]}\0" + key << "#{input[:query]}\0" key << "#{input[:drilldown_output_columns]}\0" key << "#{input[:drilldown_offset]}\0" key << "#{input[:drilldown_limit]}\0" @@ -266,6 +270,8 @@ module Groonga include KeysParsable attr_reader :enumerator + attr_reader :match_columns + attr_reader :query attr_reader :filter attr_reader :offset attr_reader :limit @@ -277,9 +283,12 @@ module Groonga attr_reader :plain_drilldown attr_reader :labeled_drilldowns attr_reader :temporary_tables + attr_reader :expressions def initialize(input) @input = input @enumerator = LogicalEnumerator.new("logical_select", @input) + @match_columns = @input[:match_columns] + @query = @input[:query] @filter = @input[:filter] @offset = (@input[:offset] || 0).to_i @limit = (@input[:limit] || 10).to_i @@ -295,6 +304,8 @@ module Groonga @labeled_drilldowns = LabeledDrilldowns.parse(@input) @temporary_tables = [] + + @expressions = [] end def close @@ -313,6 +324,10 @@ module Groonga @temporary_tables.each do |table| table.close end + + @expressions.each do |expression| + expression.close + end end end @@ -396,11 +411,12 @@ module Groonga def close end - def apply(table) + def apply(table, condition=nil) column = table.create_column(@label, @flags, @type) expression = Expression.create(table) begin expression.parse(@value) + expression.condition = condition if condition table.apply_expression(column, expression) ensure expression.close @@ -700,6 +716,8 @@ module Groonga @target_table =****@shard***** + @match_columns =****@conte*****_columns + @query =****@conte***** @filter =****@conte***** @sort_keys =****@conte*****_keys @result_sets =****@conte*****_sets @@ -721,41 +739,38 @@ module Groonga raise InvalidArgument, message end - expression_builder = RangeExpressionBuilder.new(shard_key, - @target_range, - @filter) @context.dynamic_columns.each_initial do |dynamic_column| if @target_table ==****@shard***** - create_expression(@target_table) do |expression| - expression.append_constant(true, Operator::PUSH, 1) - @target_table = @target_table.select(expression) - end + @target_table = create_all_match_table(@target_table) + @context.temporary_tables << @target_table end dynamic_column.apply(@target_table) end - case @cover_type - when :all - filter_shard_all(expression_builder) - when :partial_min - filter_table do |expression| - expression_builder.build_partial_min(expression) - end - when :partial_max - filter_table do |expression| - expression_builder.build_partial_max(expression) - end - when :partial_min_and_max - filter_table do |expression| - expression_builder.build_partial_min_and_max(expression) + create_expression_builder(shard_key) do |expression_builder| + case @cover_type + when :all + filter_shard_all(expression_builder) + when :partial_min + filter_table do |expression| + expression_builder.build_partial_min(expression) + end + when :partial_max + filter_table do |expression| + expression_builder.build_partial_max(expression) + end + when :partial_min_and_max + filter_table do |expression| + expression_builder.build_partial_min_and_max(expression) + end end end end private def filter_shard_all(expression_builder) - if****@filte*****? - add_result_set(@target_table) + if****@query*****? and****@filte*****? + add_result_set(@target_table, nil) else filter_table do |expression| expression_builder.build_all(expression) @@ -765,22 +780,32 @@ module Groonga def create_expression(table) expression = Expression.create(table) + @context.expressions << expression + expression + end + + def create_expression_builder(shard_key) + expression_builder = RangeExpressionBuilder.new(shard_key, + @target_range) + expression_builder.match_columns = @match_columns + expression_builder.query = @query + expression_builder.filter = @filter begin - yield(expression) + yield(expression_builder) ensure - expression.close + expression = expression_builder.match_columns_expression + @context.expressions << expression if expression end end def filter_table table = @target_table - create_expression(table) do |expression| - yield(expression) - add_result_set(table.select(expression)) - end + expression = create_expression(table) + yield(expression) + add_result_set(table.select(expression), expression) end - def add_result_set(result_set) + def add_result_set(result_set, condition) if result_set.empty? result_set.close return @@ -794,6 +819,16 @@ module Groonga @result_sets << sorted_result_set end end + + def create_all_match_table(table) + expression = Expression.create(table) + begin + expression.append_constant(true, Operator::PUSH, 1) + table.select(expression) + ensure + expression.close + end + end end end end Modified: plugins/sharding/logical_table_remove.rb (+1 -2) =================================================================== --- plugins/sharding/logical_table_remove.rb 2017-03-16 14:46:40 +0900 (6cd0f4a) +++ plugins/sharding/logical_table_remove.rb 2017-03-16 14:48:33 +0900 (3353d6c) @@ -50,8 +50,7 @@ module Groonga end expression_builder = RangeExpressionBuilder.new(shard_key, - target_range, - nil) + target_range) case cover_type when :partial_min remove_records(table) do |expression| Modified: plugins/sharding/range_expression_builder.rb (+37 -14) =================================================================== --- plugins/sharding/range_expression_builder.rb 2017-03-16 14:46:40 +0900 (2e2dd29) +++ plugins/sharding/range_expression_builder.rb 2017-03-16 14:48:33 +0900 (cc80735) @@ -1,16 +1,23 @@ module Groonga module Sharding class RangeExpressionBuilder - def initialize(key, target_range, filter) + attr_reader :match_columns_expression + + attr_writer :match_columns + attr_writer :query + attr_writer :filter + + def initialize(key, target_range) @key = key @target_range = target_range - @filter = filter + @match_columns_expression = nil + @match_columns = nil + @query = nil + @filter = nil end def build_all(expression) - return if****@filte*****? - - expression.parse(@filter) + build_condition(expression) end def build_partial_min(expression) @@ -22,10 +29,7 @@ module Groonga else expression.append_operator(Operator::GREATER, 2) end - if @filter - expression.parse(@filter) - expression.append_operator(Operator::AND, 2) - end + build_condition(expression) end def build_partial_max(expression) @@ -37,10 +41,7 @@ module Groonga else expression.append_operator(Operator::LESS, 2) end - if @filter - expression.parse(@filter) - expression.append_operator(Operator::AND, 2) - end + build_condition(expression) end def build_partial_min_and_max(expression) @@ -55,9 +56,31 @@ module Groonga expression.append_constant(@target_range.max_border, Operator::PUSH, 1) expression.append_operator(Operator::CALL, 5) + build_condition(expression) + end + + private + def build_condition(expression) + if @query + is_empty = expression.empty? + if @match_columns + table = Context.instance[expression[0].domain] + @match_columns_expression = Expression.create(table) + @match_columns_expression.parse(@match_columns) + end + flags = Expression::SYNTAX_QUERY | + Expression::ALLOW_PRAGMA | + Expression::ALLOW_COLUMN + expression.parse(@query, + default_column: @match_columns_expression, + flags: flags) + expression.append_operator(Operator::AND, 2) unless is_empty + end + if @filter + is_empty = expression.empty? expression.parse(@filter) - expression.append_operator(Operator::AND, 2) + expression.append_operator(Operator::AND, 2) unless is_empty end end end Added: test/command/suite/sharding/logical_select/query/column.expected (+78 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_select/query/column.expected 2017-03-16 14:48:33 +0900 (721ba24) @@ -0,0 +1,78 @@ +plugin_register sharding +[[0,0.0,0.0],true] +plugin_register functions/number +[[0,0.0,0.0],true] +table_create Memos_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Memos_20170315 content COLUMN_SCALAR Text +[[0,0.0,0.0],true] +table_create Memos_20170316 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos_20170316 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Memos_20170316 content COLUMN_SCALAR Text +[[0,0.0,0.0],true] +table_create Memos_20170317 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos_20170317 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Memos_20170317 content COLUMN_SCALAR Text +[[0,0.0,0.0],true] +load --table Memos_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "content": "Groonga is fast."}, +{"timestamp": "2017/03/15 01:00:00", "content": "Mroonga is fast and easy to use."} +] +[[0,0.0,0.0],2] +load --table Memos_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "content": "PGroonga is fast and easy to use."}, +{"timestamp": "2017/03/16 11:00:00", "content": "Rroonga is fast and easy to use."} +] +[[0,0.0,0.0],2] +logical_select Memos --shard_key timestamp --query 'content:@easy' +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + 3 + ], + [ + [ + "_id", + "UInt32" + ], + [ + "content", + "Text" + ], + [ + "timestamp", + "Time" + ] + ], + [ + 2, + "Mroonga is fast and easy to use.", + 1489507200.0 + ], + [ + 1, + "PGroonga is fast and easy to use.", + 1489626000.0 + ], + [ + 2, + "Rroonga is fast and easy to use.", + 1489629600.0 + ] + ] + ] +] Added: test/command/suite/sharding/logical_select/query/column.test (+33 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_select/query/column.test 2017-03-16 14:48:33 +0900 (52d66f6) @@ -0,0 +1,33 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +plugin_register functions/number + +table_create Memos_20170315 TABLE_NO_KEY +column_create Memos_20170315 timestamp COLUMN_SCALAR Time +column_create Memos_20170315 content COLUMN_SCALAR Text + +table_create Memos_20170316 TABLE_NO_KEY +column_create Memos_20170316 timestamp COLUMN_SCALAR Time +column_create Memos_20170316 content COLUMN_SCALAR Text + +table_create Memos_20170317 TABLE_NO_KEY +column_create Memos_20170317 timestamp COLUMN_SCALAR Time +column_create Memos_20170317 content COLUMN_SCALAR Text + +load --table Memos_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "content": "Groonga is fast."}, +{"timestamp": "2017/03/15 01:00:00", "content": "Mroonga is fast and easy to use."} +] + +load --table Memos_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "content": "PGroonga is fast and easy to use."}, +{"timestamp": "2017/03/16 11:00:00", "content": "Rroonga is fast and easy to use."} +] + +logical_select Memos \ + --shard_key timestamp \ + --query 'content:@easy' Added: test/command/suite/sharding/logical_select/query/with_filter.expected (+68 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_select/query/with_filter.expected 2017-03-16 14:48:33 +0900 (3ea6084) @@ -0,0 +1,68 @@ +plugin_register sharding +[[0,0.0,0.0],true] +plugin_register functions/number +[[0,0.0,0.0],true] +table_create Memos_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Memos_20170315 content COLUMN_SCALAR Text +[[0,0.0,0.0],true] +table_create Memos_20170316 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos_20170316 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Memos_20170316 content COLUMN_SCALAR Text +[[0,0.0,0.0],true] +table_create Memos_20170317 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos_20170317 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Memos_20170317 content COLUMN_SCALAR Text +[[0,0.0,0.0],true] +load --table Memos_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "content": "Groonga is fast."}, +{"timestamp": "2017/03/15 01:00:00", "content": "Mroonga is fast and easy to use."} +] +[[0,0.0,0.0],2] +load --table Memos_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "content": "PGroonga is fast and easy to use."}, +{"timestamp": "2017/03/16 11:00:00", "content": "Rroonga is fast and easy to use."} +] +[[0,0.0,0.0],2] +logical_select Memos --shard_key timestamp --query 'content:@easy' --filter 'content @ "Groonga"' +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + 1 + ], + [ + [ + "_id", + "UInt32" + ], + [ + "content", + "Text" + ], + [ + "timestamp", + "Time" + ] + ], + [ + 1, + "PGroonga is fast and easy to use.", + 1489626000.0 + ] + ] + ] +] Added: test/command/suite/sharding/logical_select/query/with_filter.test (+34 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_select/query/with_filter.test 2017-03-16 14:48:33 +0900 (23418c0) @@ -0,0 +1,34 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +plugin_register functions/number + +table_create Memos_20170315 TABLE_NO_KEY +column_create Memos_20170315 timestamp COLUMN_SCALAR Time +column_create Memos_20170315 content COLUMN_SCALAR Text + +table_create Memos_20170316 TABLE_NO_KEY +column_create Memos_20170316 timestamp COLUMN_SCALAR Time +column_create Memos_20170316 content COLUMN_SCALAR Text + +table_create Memos_20170317 TABLE_NO_KEY +column_create Memos_20170317 timestamp COLUMN_SCALAR Time +column_create Memos_20170317 content COLUMN_SCALAR Text + +load --table Memos_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "content": "Groonga is fast."}, +{"timestamp": "2017/03/15 01:00:00", "content": "Mroonga is fast and easy to use."} +] + +load --table Memos_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "content": "PGroonga is fast and easy to use."}, +{"timestamp": "2017/03/16 11:00:00", "content": "Rroonga is fast and easy to use."} +] + +logical_select Memos \ + --shard_key timestamp \ + --query 'content:@easy' \ + --filter 'content @ "Groonga"' Added: test/command/suite/sharding/logical_select/query/with_match_columns.expected (+78 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_select/query/with_match_columns.expected 2017-03-16 14:48:33 +0900 (ed22c21) @@ -0,0 +1,78 @@ +plugin_register sharding +[[0,0.0,0.0],true] +plugin_register functions/number +[[0,0.0,0.0],true] +table_create Memos_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Memos_20170315 content COLUMN_SCALAR Text +[[0,0.0,0.0],true] +table_create Memos_20170316 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos_20170316 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Memos_20170316 content COLUMN_SCALAR Text +[[0,0.0,0.0],true] +table_create Memos_20170317 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos_20170317 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Memos_20170317 content COLUMN_SCALAR Text +[[0,0.0,0.0],true] +load --table Memos_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "content": "Groonga is fast."}, +{"timestamp": "2017/03/15 01:00:00", "content": "Mroonga is fast and easy to use."} +] +[[0,0.0,0.0],2] +load --table Memos_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "content": "PGroonga is fast and easy to use."}, +{"timestamp": "2017/03/16 11:00:00", "content": "Rroonga is fast and easy to use."} +] +[[0,0.0,0.0],2] +logical_select Memos --shard_key timestamp --match_columns 'content' --query 'easy' +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + 3 + ], + [ + [ + "_id", + "UInt32" + ], + [ + "content", + "Text" + ], + [ + "timestamp", + "Time" + ] + ], + [ + 2, + "Mroonga is fast and easy to use.", + 1489507200.0 + ], + [ + 1, + "PGroonga is fast and easy to use.", + 1489626000.0 + ], + [ + 2, + "Rroonga is fast and easy to use.", + 1489629600.0 + ] + ] + ] +] Added: test/command/suite/sharding/logical_select/query/with_match_columns.test (+34 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_select/query/with_match_columns.test 2017-03-16 14:48:33 +0900 (c60aeee) @@ -0,0 +1,34 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +plugin_register functions/number + +table_create Memos_20170315 TABLE_NO_KEY +column_create Memos_20170315 timestamp COLUMN_SCALAR Time +column_create Memos_20170315 content COLUMN_SCALAR Text + +table_create Memos_20170316 TABLE_NO_KEY +column_create Memos_20170316 timestamp COLUMN_SCALAR Time +column_create Memos_20170316 content COLUMN_SCALAR Text + +table_create Memos_20170317 TABLE_NO_KEY +column_create Memos_20170317 timestamp COLUMN_SCALAR Time +column_create Memos_20170317 content COLUMN_SCALAR Text + +load --table Memos_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "content": "Groonga is fast."}, +{"timestamp": "2017/03/15 01:00:00", "content": "Mroonga is fast and easy to use."} +] + +load --table Memos_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "content": "PGroonga is fast and easy to use."}, +{"timestamp": "2017/03/16 11:00:00", "content": "Rroonga is fast and easy to use."} +] + +logical_select Memos \ + --shard_key timestamp \ + --match_columns 'content' \ + --query 'easy' -------------- next part -------------- HTML����������������������������...Download