Groonga - fulltext search engine.

[Groonga-commit] groonga/groonga at a004523 [master] logical_select: support drilldown filter

Kouhei Sutou null+****@clear*****
Tue Mar 21 20:38:34 JST 2017

• Previous message: [Groonga-commit] Still Failing: groonga/groonga#8728 (master - 8b1daf3)
• Next message: [Groonga-commit] groonga/groonga at 4b4cbb3 [master] logical_select: update cache key for dynamic columns in drilldown
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

Kouhei Sutou	2017-03-21 20:38:34 +0900 (Tue, 21 Mar 2017)

  New Revision: a004523e3ec04a08df896fd56115c281f7d78b01
  https://github.com/groonga/groonga/commit/a004523e3ec04a08df896fd56115c281f7d78b01

  Message:
    logical_select: support drilldown filter

  Added files:
    test/command/suite/sharding/logical_select/drilldown/plain/filter/key.expected
    test/command/suite/sharding/logical_select/drilldown/plain/filter/key.test
    test/command/suite/sharding/logical_select/drilldowns/filter/key.expected
    test/command/suite/sharding/logical_select/drilldowns/filter/key.test
  Modified files:
    plugins/sharding/logical_select.rb

  Modified: plugins/sharding/logical_select.rb (+49 -0)
===================================================================
--- plugins/sharding/logical_select.rb    2017-03-21 19:48:01 +0900 (3eea6a2)
+++ plugins/sharding/logical_select.rb    2017-03-21 20:38:34 +0900 (46bc8e1)
@@ -27,6 +27,7 @@ module Groonga
                  "drilldown_sort_keys",
                  "match_columns",
                  "query",
+                 "drilldown_filter",
                ])
 
       def run_body(input)
@@ -85,6 +86,7 @@ module Groonga
         key << "#{input[:drilldown_limit]}\0"
         key << "#{input[:drilldown_calc_types]}\0"
         key << "#{input[:drilldown_calc_target]}\0"
+        key << "#{input[:drilldown_filter]}\0"
         labeled_drilldowns = LabeledDrilldowns.parse(input).sort_by(&:label)
         labeled_drilldowns.each do |drilldown|
           key << "#{drilldown.label}\0"
@@ -94,6 +96,7 @@ module Groonga
           key << "#{drilldown.limit}\0"
           key << "#{drilldown.calc_types}\0"
           key << "#{drilldown.calc_target_name}\0"
+          key << "#{drilldown.filter}\0"
         end
         dynamic_columns = DynamicColumns.parse(input)
         [
@@ -481,8 +484,11 @@ module Groonga
         attr_reader :output_columns
         attr_reader :calc_target_name
         attr_reader :calc_types
+        attr_reader :filter
         attr_reader :result_sets
         attr_reader :unsorted_result_sets
+        attr_reader :temporary_tables
+        attr_reader :expressions
         def initialize(input)
           @input = input
           @keys = parse_keys(@input[:drilldown])
@@ -494,9 +500,14 @@ module Groonga
           @output_columns ||= "_key, _nsubrecs"
           @calc_target_name = @input[:drilldown_calc_target]
           @calc_types = parse_calc_types(@input[:drilldown_calc_types])
+          @filter = @input[:drilldown_filter]
 
           @result_sets = []
           @unsorted_result_sets = []
+
+          @temporary_tables = []
+
+          @expressions = []
         end
 
         def close
@@ -506,6 +517,14 @@ module Groonga
           @unsorted_result_sets.each do |result_set|
             result_set.close
           end
+
+          @temporary_tables.each do |table|
+            table.close
+          end
+
+          @expressions.each do |expression|
+            expression.close
+          end
         end
 
         def have_keys?
@@ -596,10 +615,13 @@ module Groonga
         attr_reader :output_columns
         attr_reader :calc_target_name
         attr_reader :calc_types
+        attr_reader :filter
         attr_reader :table
         attr_reader :dynamic_columns
         attr_accessor :result_set
         attr_accessor :unsorted_result_set
+        attr_reader :temporary_tables
+        attr_reader :expressions
         def initialize(label, parameters)
           @label = label
           @keys = parse_keys(parameters["keys"])
@@ -611,12 +633,17 @@ module Groonga
           @output_columns ||= "_key, _nsubrecs"
           @calc_target_name = parameters["calc_target"]
           @calc_types = parse_calc_types(parameters["calc_types"])
+          @filter = parameters["filter"]
           @table = parameters["table"]
 
           @dynamic_columns = DynamicColumns.parse(parameters)
 
           @result_set = nil
           @unsorted_result_set = nil
+
+          @temporary_tables = []
+
+          @expressions = []
         end
 
         def close
@@ -624,6 +651,14 @@ module Groonga
           @unsorted_result_set.close if @unsorted_result_set
 
           @dynamic_columns.close
+
+          @temporary_tables.each do |table|
+            table.close
+          end
+
+          @expressions.each do |expression|
+            expression.close
+          end
         end
 
         def need_command_version2?
@@ -702,6 +737,7 @@ module Groonga
                 end
               end
               result_set = group_result.table
+              result_set = apply_drilldown_filter(drilldown, result_set)
               if drilldown.sort_keys.empty?
                 drilldown.result_sets << result_set
               else
@@ -747,6 +783,7 @@ module Groonga
               drilldown.dynamic_columns.each_initial do |dynamic_column|
                 dynamic_column.apply(result_set)
               end
+              result_set = apply_drilldown_filter(drilldown, result_set)
               if drilldown.sort_keys.empty?
                 drilldown.result_set = result_set
               else
@@ -769,6 +806,18 @@ module Groonga
             group_result.calc_target = nil
           end
         end
+
+        def apply_drilldown_filter(drilldown, result_set)
+          filter = drilldown.filter
+          return result_set if filter.nil?
+
+          expression = Expression.create(result_set)
+          drilldown.expressions << expression
+          expression.parse(filter)
+          filtered_result_set = result_set.select(expression)
+          drilldown.temporary_tables << result_set
+          filtered_result_set
+        end
       end
 
       class ShardExecutor

  Added: test/command/suite/sharding/logical_select/drilldown/plain/filter/key.expected (+135 -0) 100644
===================================================================
--- /dev/null
+++ test/command/suite/sharding/logical_select/drilldown/plain/filter/key.expected    2017-03-21 20:38:34 +0900 (78edfd1)
@@ -0,0 +1,135 @@
+plugin_register sharding
+[[0,0.0,0.0],true]
+table_create Logs_20150203 TABLE_NO_KEY
+[[0,0.0,0.0],true]
+column_create Logs_20150203 timestamp COLUMN_SCALAR Time
+[[0,0.0,0.0],true]
+column_create Logs_20150203 memo COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+column_create Logs_20150203 action COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+table_create Logs_20150204 TABLE_NO_KEY
+[[0,0.0,0.0],true]
+column_create Logs_20150204 timestamp COLUMN_SCALAR Time
+[[0,0.0,0.0],true]
+column_create Logs_20150204 memo COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+column_create Logs_20150204 action COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+table_create Logs_20150205 TABLE_NO_KEY
+[[0,0.0,0.0],true]
+column_create Logs_20150205 timestamp COLUMN_SCALAR Time
+[[0,0.0,0.0],true]
+column_create Logs_20150205 memo COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+column_create Logs_20150205 action COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+load --table Logs_20150203
+[
+{
+  "timestamp": "2015-02-03 12:49:00",
+  "memo":      "2015-02-03 12:49:00",
+  "action":    "Start"
+},
+{
+  "timestamp": "2015-02-03 23:59:59",
+  "memo":      "2015-02-03 23:59:59",
+  "action":    "Shutdown"
+}
+]
+[[0,0.0,0.0],2]
+load --table Logs_20150204
+[
+{
+  "timestamp": "2015-02-04 00:00:00",
+  "memo":      "2015-02-04 00:00:00",
+  "action":    "Start"
+},
+{
+  "timestamp": "2015-02-04 13:49:00",
+  "memo":      "2015-02-04 13:49:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-04 13:50:00",
+  "memo":      "2015-02-04 13:50:00",
+  "action":    "Restart"
+}
+]
+[[0,0.0,0.0],3]
+load --table Logs_20150205
+[
+{
+  "timestamp": "2015-02-05 13:49:00",
+  "memo":      "2015-02-05 13:49:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-05 13:50:00",
+  "memo":      "2015-02-05 13:50:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-05 13:51:00",
+  "memo":      "2015-02-05 13:51:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-05 13:52:00",
+  "memo":      "2015-02-05 13:52:00",
+  "action":    "Restart"
+}
+]
+[[0,0.0,0.0],4]
+logical_select Logs timestamp   --limit 0   --drilldown action   --drilldown_filter '_key == "Restart"'
+[
+  [
+    0,
+    0.0,
+    0.0
+  ],
+  [
+    [
+      [
+        9
+      ],
+      [
+        [
+          "_id",
+          "UInt32"
+        ],
+        [
+          "action",
+          "ShortText"
+        ],
+        [
+          "memo",
+          "ShortText"
+        ],
+        [
+          "timestamp",
+          "Time"
+        ]
+      ]
+    ],
+    [
+      [
+        1
+      ],
+      [
+        [
+          "_key",
+          "ShortText"
+        ],
+        [
+          "_nsubrecs",
+          "Int32"
+        ]
+      ],
+      [
+        "Restart",
+        6
+      ]
+    ]
+  ]
+]

  Added: test/command/suite/sharding/logical_select/drilldown/plain/filter/key.test (+80 -0) 100644
===================================================================
--- /dev/null
+++ test/command/suite/sharding/logical_select/drilldown/plain/filter/key.test    2017-03-21 20:38:34 +0900 (9878ce0)
@@ -0,0 +1,80 @@
+#@on-error omit
+plugin_register sharding
+#@on-error default
+
+table_create Logs_20150203 TABLE_NO_KEY
+column_create Logs_20150203 timestamp COLUMN_SCALAR Time
+column_create Logs_20150203 memo COLUMN_SCALAR ShortText
+column_create Logs_20150203 action COLUMN_SCALAR ShortText
+
+table_create Logs_20150204 TABLE_NO_KEY
+column_create Logs_20150204 timestamp COLUMN_SCALAR Time
+column_create Logs_20150204 memo COLUMN_SCALAR ShortText
+column_create Logs_20150204 action COLUMN_SCALAR ShortText
+
+table_create Logs_20150205 TABLE_NO_KEY
+column_create Logs_20150205 timestamp COLUMN_SCALAR Time
+column_create Logs_20150205 memo COLUMN_SCALAR ShortText
+column_create Logs_20150205 action COLUMN_SCALAR ShortText
+
+load --table Logs_20150203
+[
+{
+  "timestamp": "2015-02-03 12:49:00",
+  "memo":      "2015-02-03 12:49:00",
+  "action":    "Start"
+},
+{
+  "timestamp": "2015-02-03 23:59:59",
+  "memo":      "2015-02-03 23:59:59",
+  "action":    "Shutdown"
+}
+]
+
+load --table Logs_20150204
+[
+{
+  "timestamp": "2015-02-04 00:00:00",
+  "memo":      "2015-02-04 00:00:00",
+  "action":    "Start"
+},
+{
+  "timestamp": "2015-02-04 13:49:00",
+  "memo":      "2015-02-04 13:49:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-04 13:50:00",
+  "memo":      "2015-02-04 13:50:00",
+  "action":    "Restart"
+}
+]
+
+load --table Logs_20150205
+[
+{
+  "timestamp": "2015-02-05 13:49:00",
+  "memo":      "2015-02-05 13:49:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-05 13:50:00",
+  "memo":      "2015-02-05 13:50:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-05 13:51:00",
+  "memo":      "2015-02-05 13:51:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-05 13:52:00",
+  "memo":      "2015-02-05 13:52:00",
+  "action":    "Restart"
+}
+]
+
+logical_select Logs timestamp \
+  --limit 0 \
+  --drilldown action \
+  --drilldown_filter '_key == "Restart"'

  Added: test/command/suite/sharding/logical_select/drilldowns/filter/key.expected (+137 -0) 100644
===================================================================
--- /dev/null
+++ test/command/suite/sharding/logical_select/drilldowns/filter/key.expected    2017-03-21 20:38:34 +0900 (e6d9555)
@@ -0,0 +1,137 @@
+plugin_register sharding
+[[0,0.0,0.0],true]
+table_create Logs_20150203 TABLE_NO_KEY
+[[0,0.0,0.0],true]
+column_create Logs_20150203 timestamp COLUMN_SCALAR Time
+[[0,0.0,0.0],true]
+column_create Logs_20150203 memo COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+column_create Logs_20150203 action COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+table_create Logs_20150204 TABLE_NO_KEY
+[[0,0.0,0.0],true]
+column_create Logs_20150204 timestamp COLUMN_SCALAR Time
+[[0,0.0,0.0],true]
+column_create Logs_20150204 memo COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+column_create Logs_20150204 action COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+table_create Logs_20150205 TABLE_NO_KEY
+[[0,0.0,0.0],true]
+column_create Logs_20150205 timestamp COLUMN_SCALAR Time
+[[0,0.0,0.0],true]
+column_create Logs_20150205 memo COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+column_create Logs_20150205 action COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+load --table Logs_20150203
+[
+{
+  "timestamp": "2015-02-03 12:49:00",
+  "memo":      "2015-02-03 12:49:00",
+  "action":    "Start"
+},
+{
+  "timestamp": "2015-02-03 23:59:59",
+  "memo":      "2015-02-03 23:59:59",
+  "action":    "Shutdown"
+}
+]
+[[0,0.0,0.0],2]
+load --table Logs_20150204
+[
+{
+  "timestamp": "2015-02-04 00:00:00",
+  "memo":      "2015-02-04 00:00:00",
+  "action":    "Start"
+},
+{
+  "timestamp": "2015-02-04 13:49:00",
+  "memo":      "2015-02-04 13:49:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-04 13:50:00",
+  "memo":      "2015-02-04 13:50:00",
+  "action":    "Restart"
+}
+]
+[[0,0.0,0.0],3]
+load --table Logs_20150205
+[
+{
+  "timestamp": "2015-02-05 13:49:00",
+  "memo":      "2015-02-05 13:49:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-05 13:50:00",
+  "memo":      "2015-02-05 13:50:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-05 13:51:00",
+  "memo":      "2015-02-05 13:51:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-05 13:52:00",
+  "memo":      "2015-02-05 13:52:00",
+  "action":    "Restart"
+}
+]
+[[0,0.0,0.0],4]
+logical_select Logs timestamp   --limit 0   --drilldowns[action].keys action   --drilldowns[action].filter '_key == "Restart"'
+[
+  [
+    0,
+    0.0,
+    0.0
+  ],
+  [
+    [
+      [
+        9
+      ],
+      [
+        [
+          "_id",
+          "UInt32"
+        ],
+        [
+          "action",
+          "ShortText"
+        ],
+        [
+          "memo",
+          "ShortText"
+        ],
+        [
+          "timestamp",
+          "Time"
+        ]
+      ]
+    ],
+    {
+      "action": [
+        [
+          1
+        ],
+        [
+          [
+            "_key",
+            "ShortText"
+          ],
+          [
+            "_nsubrecs",
+            "Int32"
+          ]
+        ],
+        [
+          "Restart",
+          6
+        ]
+      ]
+    }
+  ]
+]

  Added: test/command/suite/sharding/logical_select/drilldowns/filter/key.test (+80 -0) 100644
===================================================================
--- /dev/null
+++ test/command/suite/sharding/logical_select/drilldowns/filter/key.test    2017-03-21 20:38:34 +0900 (856741f)
@@ -0,0 +1,80 @@
+#@on-error omit
+plugin_register sharding
+#@on-error default
+
+table_create Logs_20150203 TABLE_NO_KEY
+column_create Logs_20150203 timestamp COLUMN_SCALAR Time
+column_create Logs_20150203 memo COLUMN_SCALAR ShortText
+column_create Logs_20150203 action COLUMN_SCALAR ShortText
+
+table_create Logs_20150204 TABLE_NO_KEY
+column_create Logs_20150204 timestamp COLUMN_SCALAR Time
+column_create Logs_20150204 memo COLUMN_SCALAR ShortText
+column_create Logs_20150204 action COLUMN_SCALAR ShortText
+
+table_create Logs_20150205 TABLE_NO_KEY
+column_create Logs_20150205 timestamp COLUMN_SCALAR Time
+column_create Logs_20150205 memo COLUMN_SCALAR ShortText
+column_create Logs_20150205 action COLUMN_SCALAR ShortText
+
+load --table Logs_20150203
+[
+{
+  "timestamp": "2015-02-03 12:49:00",
+  "memo":      "2015-02-03 12:49:00",
+  "action":    "Start"
+},
+{
+  "timestamp": "2015-02-03 23:59:59",
+  "memo":      "2015-02-03 23:59:59",
+  "action":    "Shutdown"
+}
+]
+
+load --table Logs_20150204
+[
+{
+  "timestamp": "2015-02-04 00:00:00",
+  "memo":      "2015-02-04 00:00:00",
+  "action":    "Start"
+},
+{
+  "timestamp": "2015-02-04 13:49:00",
+  "memo":      "2015-02-04 13:49:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-04 13:50:00",
+  "memo":      "2015-02-04 13:50:00",
+  "action":    "Restart"
+}
+]
+
+load --table Logs_20150205
+[
+{
+  "timestamp": "2015-02-05 13:49:00",
+  "memo":      "2015-02-05 13:49:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-05 13:50:00",
+  "memo":      "2015-02-05 13:50:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-05 13:51:00",
+  "memo":      "2015-02-05 13:51:00",
+  "action":    "Restart"
+},
+{
+  "timestamp": "2015-02-05 13:52:00",
+  "memo":      "2015-02-05 13:52:00",
+  "action":    "Restart"
+}
+]
+
+logical_select Logs timestamp \
+  --limit 0 \
+  --drilldowns[action].keys action \
+  --drilldowns[action].filter '_key == "Restart"'
-------------- next part --------------
HTML����������������������������...
Download 

• Previous message: [Groonga-commit] Still Failing: groonga/groonga#8728 (master - 8b1daf3)
• Next message: [Groonga-commit] groonga/groonga at 4b4cbb3 [master] logical_select: update cache key for dynamic columns in drilldown
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]