Kouhei Sutou
null+****@clear*****
Wed Nov 5 15:44:47 JST 2014
Kouhei Sutou 2014-11-05 15:44:47 +0900 (Wed, 05 Nov 2014) New Revision: ef29c3940f608096a60672b2ceb44adeda1c1640 https://github.com/groonga/fluent-plugin-groonga/commit/ef29c3940f608096a60672b2ceb44adeda1c1640 Message: out: support <mapping> It is for define type and indexes. Modified files: lib/fluent/plugin/out_groonga.rb sample/store-apache.conf Modified: lib/fluent/plugin/out_groonga.rb (+29 -13) =================================================================== --- lib/fluent/plugin/out_groonga.rb 2014-11-05 15:39:10 +0900 (2395fdc) +++ lib/fluent/plugin/out_groonga.rb 2014-11-05 15:44:47 +0900 (26a4b9b) @@ -82,7 +82,8 @@ module Fluent @client = create_client(@protocol) @client.configure(conf) - @emitter = Emitter.new(@client, @store_table) + @schema = Schema.new(@client, @store_table, @mappings) + @emitter = Emitter.new(@client, @store_table, @schema) @tables =****@table***** do |table| TableDefinition.new(table) @@ -264,13 +265,12 @@ module Fluent table.name == definition.name end if existing_table - if definition.have_difference?(existing_table) - # TODO: Is it OK? - @client.execute("table_remove", "name" => definition.name) - end - @client.execute("table_create", definition.to_create_arguments) + next unless definition.have_difference?(existing_table) + # TODO: Is it OK? + @client.execute("table_remove", "name" => definition.name) end + @client.execute("table_create", definition.to_create_arguments) definition.indexes.each do |index| @client.execute("column_create", index.to_create_arguments) end @@ -279,9 +279,10 @@ module Fluent end class Schema - def initialize(client, table_name) + def initialize(client, table_name, mappings) @client = client @table_name = table_name + @mappings = mappings @table = nil @columns = nil end @@ -317,7 +318,6 @@ module Fluent if target_table @table = Table.new(@table_name, target_table.domain) else - # TODO: Check response @client.execute("table_create", "name" => @table_name, "flags" => "TABLE_NO_KEY") @@ -339,20 +339,37 @@ module Fluent end def create_column(name, sample_values) + mapping =****@mappi***** do |mapping| + mapping.name == name + end + if mapping + value_type = mapping[:type] + end guesser = TypeGuesser.new(sample_values) - value_type = guesser.guess + value_type ||= guesser.guess vector_p = guesser.vector? if vector_p flags = "COLUMN_VECTOR" else flags = "COLUMN_SCALAR" end - # TODO: Check response @client.execute("column_create", "table" => @table_name, "name" => name, "flags" => flags, "type" => value_type) + if mapping + mapping.indexes.each do |index| + index_flags = ["COLUMN_INDEX", index[:flags]].compact + @client.execute("column_create", + "table" => index[:table], + "name" => index[:table], + "flags" => index_flags.join("|"), + "type" => @table_name, + "source" => name) + end + end + Column.new(name, value_type, vector_p) end @@ -470,14 +487,13 @@ module Fluent end class Emitter - def initialize(client, table) + def initialize(client, table, schema) @client = client @table = table - @schema = nil + @schema = schema end def start - @schema = Schema.new(@client, @table) end def shutdown Modified: sample/store-apache.conf (+102 -1) =================================================================== --- sample/store-apache.conf 2014-11-05 15:39:10 +0900 (e86110c) +++ sample/store-apache.conf 2014-11-05 15:44:47 +0900 (0db881b) @@ -8,7 +8,7 @@ pos_file /tmp/apache_access.pos tag apache.raw.log.apache.access format apache2 - read_from_head "#{ENV['FLUENT_PLUGIN_GROONGA_DEBUG'] == 'yes'}" + # read_from_head true </source> <match apache.**> @@ -135,4 +135,105 @@ source_columns _key </index> </table> + + <mapping> + name agent + type UserAgents + <index> + table Terms + name logs_agent_index + flags WITH_POSITION + </index> + </mapping> + + <mapping> + name code + type Codes + <index> + table Codes + name logs_index + </index> + </mapping> + + <mapping> + name host + type Hosts + <index> + table Hosts + name hosts_index + </index> + </mapping> + + <mapping> + name message + type Text + <index> + table Terms + name logs_message_index + flags WITH_POSITION + </index> + </mapping> + + <mapping> + name method + type Methods + <index> + table Methods + name logs_index + </index> + </mapping> + + <mapping> + name path + type Paths + <index> + table Paths + name logs_index + </index> + </mapping> + + <mapping> + name referer + type URLs + <index> + table URLs + name logs_index + </index> + </mapping> + + <mapping> + name remote + type Remotes + <index> + table Remotes + name logs_index + </index> + </mapping> + + <mapping> + name size + type Int32 + <index> + table Sizes + name logs_index + </index> + </mapping> + + <mapping> + name timestamp + type Time + <index> + table Timestamps + name logs_index + </index> + </mapping> + + <mapping> + name type + type Types + <index> + table Types + name logs_index + </index> + </mapping> </match> -------------- next part -------------- HTML����������������������������...Download